diff --git a/.eslintrc.base.js b/.eslintrc.base.js index 9684533..b4ce345 100644 --- a/.eslintrc.base.js +++ b/.eslintrc.base.js @@ -109,6 +109,10 @@ module.exports = (projectRoot, extraRules = {}) => ({ "@typescript-eslint/no-useless-constructor": ["error"], "@typescript-eslint/prefer-optional-chain": ["error"], "@typescript-eslint/consistent-type-imports": ["error"], + "@typescript-eslint/require-array-sort-compare": [ + "error", + { ignoreStringArrays: true }, + ], eqeqeq: ["error"], "object-shorthand": ["error", "always"], "@typescript-eslint/unbound-method": ["error"], diff --git a/docs/classes/CiStorage.md b/docs/classes/CiStorage.md index 2c3b3ce..4b4c8a9 100644 --- a/docs/classes/CiStorage.md +++ b/docs/classes/CiStorage.md @@ -53,7 +53,7 @@ Construct.constructor #### Defined in -[src/CiStorage.ts:184](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L184) +[src/CiStorage.ts:198](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L198) ## Properties @@ -63,17 +63,17 @@ Construct.constructor #### Defined in -[src/CiStorage.ts:173](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L173) +[src/CiStorage.ts:185](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L185) ___ ### securityGroup -• `Readonly` **securityGroup**: `ISecurityGroup` +• `Readonly` **securityGroup**: `SecurityGroup` #### Defined in -[src/CiStorage.ts:174](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L174) +[src/CiStorage.ts:186](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L186) ___ @@ -83,7 +83,7 @@ ___ #### Defined in -[src/CiStorage.ts:175](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L175) +[src/CiStorage.ts:187](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L187) ___ @@ -93,7 +93,7 @@ ___ #### Defined in -[src/CiStorage.ts:176](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L176) +[src/CiStorage.ts:188](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L188) ___ @@ -110,7 +110,7 @@ ___ #### Defined in -[src/CiStorage.ts:177](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L177) +[src/CiStorage.ts:189](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L189) ___ @@ -120,7 +120,7 @@ ___ #### Defined in -[src/CiStorage.ts:178](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L178) +[src/CiStorage.ts:190](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L190) ___ @@ -130,7 +130,7 @@ ___ #### Defined in -[src/CiStorage.ts:179](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L179) +[src/CiStorage.ts:191](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L191) ___ @@ -140,27 +140,17 @@ ___ #### Defined in -[src/CiStorage.ts:180](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L180) +[src/CiStorage.ts:192](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L192) ___ -### hostInstances +### hosts -• `Readonly` **hostInstances**: `Instance`[] = `[]` +• `Readonly` **hosts**: \{ `fqdn`: `undefined` \| `string` ; `instance`: `Instance` }[] = `[]` #### Defined in -[src/CiStorage.ts:181](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L181) - -___ - -### hostVolumes - -• `Readonly` **hostVolumes**: `CfnVolume`[] = `[]` - -#### Defined in - -[src/CiStorage.ts:182](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L182) +[src/CiStorage.ts:193](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L193) ___ @@ -170,7 +160,7 @@ ___ #### Defined in -[src/CiStorage.ts:185](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L185) +[src/CiStorage.ts:199](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L199) ___ @@ -180,7 +170,7 @@ ___ #### Defined in -[src/CiStorage.ts:186](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L186) +[src/CiStorage.ts:200](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L200) ___ @@ -190,4 +180,4 @@ ___ #### Defined in -[src/CiStorage.ts:187](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L187) +[src/CiStorage.ts:201](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L201) diff --git a/docs/interfaces/CiStorageProps.md b/docs/interfaces/CiStorageProps.md index 16a5e3e..3a80d07 100644 --- a/docs/interfaces/CiStorageProps.md +++ b/docs/interfaces/CiStorageProps.md @@ -17,7 +17,7 @@ VPC to use by this construct. #### Defined in -[src/CiStorage.ts:56](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L56) +[src/CiStorage.ts:57](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L57) ___ @@ -30,19 +30,20 @@ instances. #### Defined in -[src/CiStorage.ts:59](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L59) +[src/CiStorage.ts:60](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L60) ___ -### securityGroupId +### instanceNamePrefix -• **securityGroupId**: `string` +• **instanceNamePrefix**: `string` -Id of the Security Group to set for the created instances. +All instance names (and hostname for the host instances) will be prefixed +with that value, separated by "-". #### Defined in -[src/CiStorage.ts:61](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L61) +[src/CiStorage.ts:63](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L63) ___ @@ -61,7 +62,7 @@ A Hosted Zone to register the host instances in. #### Defined in -[src/CiStorage.ts:63](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L63) +[src/CiStorage.ts:65](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L65) ___ @@ -74,7 +75,7 @@ must pre-exist. #### Defined in -[src/CiStorage.ts:71](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L71) +[src/CiStorage.ts:73](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L73) ___ @@ -86,7 +87,7 @@ Time zone for instances, example: America/Los_Angeles. #### Defined in -[src/CiStorage.ts:73](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L73) +[src/CiStorage.ts:75](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L75) ___ @@ -104,19 +105,22 @@ Configuration for self-hosted runner instances in the pool. | `ghDockerComposeDirectoryUrl` | `string` | URL of docker-compose.yml (or compose.yml) directory. The tool will sparse-checkout that directory. The format is Dockerfile-compatible: https://github.com/owner/repo[#[branch]:/directory/with/compose/] | | `imageSsmName` | `string` | SSM parameter name which holds the reference to an instance image. | | `volumeGb` | `number` | Size of the root volume. | +| `swapSizeGb?` | `number` | Size of swap file (if you need it). | +| `tmpfsMaxSizeGb?` | `number` | If set, mounts /var/lib/docker to tmpfs with the provided max size. | | `instanceRequirements` | [`InstanceRequirementsProperty`, ...InstanceRequirementsProperty[]] | The list of requirements to choose Spot Instances. | -| `scale` | \{ `onDemandPercentageAboveBaseCapacity`: `number` ; `maxActiveRunnersPercent`: \{ `periodSec`: `number` ; `value`: `number` } ; `minCapacity`: \{ `id`: `string` ; `value`: `number` ; `cron`: \{ `timeZone?`: `string` } & `CronOptions` }[] ; `maxCapacity`: `number` ; `maxInstanceLifetime`: `Duration` } | Scaling options. | +| `scale` | \{ `onDemandPercentageAboveBaseCapacity`: `number` ; `maxActiveRunnersPercent`: \{ `periodSec`: `number` ; `value`: `number` ; `scalingSteps?`: `number` } ; `minCapacity`: \{ `id`: `string` ; `value`: `number` ; `cron`: \{ `timeZone?`: `string` } & `CronOptions` }[] ; `maxCapacity`: `number` ; `maxInstanceLifetime`: `Duration` } | Scaling options. | | `scale.onDemandPercentageAboveBaseCapacity` | `number` | The percentages of On-Demand Instances and Spot Instances for your additional capacity. | -| `scale.maxActiveRunnersPercent` | \{ `periodSec`: `number` ; `value`: `number` } | Maximum percentage of active runners. If the MAX metric of number of active runners within the recent periodSec interval grows beyond this threshold, the autoscaling group will launch new instances until the percentage drops, or maxCapacity is reached. | +| `scale.maxActiveRunnersPercent` | \{ `periodSec`: `number` ; `value`: `number` ; `scalingSteps?`: `number` } | Maximum percentage of active runners. If the MAX metric of number of active runners within the recent periodSec interval grows beyond this threshold, the autoscaling group will launch new instances until the percentage drops, or maxCapacity is reached. | | `scale.maxActiveRunnersPercent.periodSec` | `number` | Calculate MAX metric within that period. The higher is the value, the slower will the capacity lower (but it doesn't affect how fast will it increase). | | `scale.maxActiveRunnersPercent.value` | `number` | Value to use for the target percentage of active (busy) runners. | +| `scale.maxActiveRunnersPercent.scalingSteps?` | `number` | Desired number of ScalingInterval items in scalingSteps. | | `scale.minCapacity` | \{ `id`: `string` ; `value`: `number` ; `cron`: \{ `timeZone?`: `string` } & `CronOptions` }[] | Minimal number of idle runners to keep, depending on the daytime. If the auto scaling group has less than this number of instances, the new instances will be created. | | `scale.maxCapacity` | `number` | Maximum total number of instances. | | `scale.maxInstanceLifetime` | `Duration` | Re-create instances time to time. | #### Defined in -[src/CiStorage.ts:75](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L75) +[src/CiStorage.ts:77](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L77) ___ @@ -133,13 +137,14 @@ runner has its localhost ports redirected to that instance. | Name | Type | Description | | :------ | :------ | :------ | | `ghDockerComposeDirectoryUrl` | `string` | URL of docker-compose.yml (or compose.yml) directory. The tool will sparse-checkout that directory. The format is Dockerfile-compatible: https://github.com/owner/repo[#[branch]:/directory/with/compose/] | +| `dockerComposeProfiles?` | `string`[] | List of profiles from docker-compose to additionally start. | | `imageSsmName` | `string` | SSM parameter name which holds the reference to an instance image. | -| `volumeIops` | `number` | IOPS of the docker volume. | -| `volumeThroughput` | `number` | Throughput of the docker volume in MiB/s. | -| `volumeGb` | `number` | Size of the docker volume. | +| `swapSizeGb?` | `number` | Size of swap file (if you need it). | +| `tmpfsMaxSizeGb?` | `number` | If set, mounts /var/lib/docker to tmpfs with the provided max size and copies it from the old instance when the instance gets replaced. | | `instanceType` | `string` | Full name of the Instance type. | | `machines` | `number` | Number of instances to create. | +| `ports` | \{ `port`: `Port` ; `description`: `string` }[] | Ports to be open in the security group for connection from all runners to the host. | #### Defined in -[src/CiStorage.ts:130](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L130) +[src/CiStorage.ts:138](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L138) diff --git a/internal/clean.sh b/internal/clean.sh index f82a4a1..ae80411 100644 --- a/internal/clean.sh +++ b/internal/clean.sh @@ -1,4 +1,4 @@ #!/bin/bash set -e -rm -rf dist yarn.lock package-lock.json pnpm-lock.yaml *.log +rm -rf dist yarn.lock package-lock.json pnpm-lock.yaml node_modules ./*.log diff --git a/package.json b/package.json index 1b1c862..2b32298 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@clickup/ci-storage-cdk", "description": "A CDK construct to deploy ci-storage infrastructure", - "version": "2.10.293", + "version": "2.10.294", "license": "MIT", "keywords": [ "cdk", @@ -15,7 +15,7 @@ "build": "$npm_execpath run clean; tsc", "dev": "tsc --watch --preserveWatchOutput", "lint": "bash internal/lint.sh", - "test": "jest", + "test": "$npm_execpath run build && jest", "docs": "bash internal/docs.sh", "clean": "rm -rf dist yarn.lock package-lock.json pnpm-lock.yaml *.log", "copy-package-to-public-dir": "copy-package-to-public-dir.sh", diff --git a/src/CiStorage.ts b/src/CiStorage.ts index 393ea3e..f2a897e 100644 --- a/src/CiStorage.ts +++ b/src/CiStorage.ts @@ -4,6 +4,7 @@ import type { CronOptions, } from "aws-cdk-lib/aws-autoscaling"; import { + AdjustmentType, AutoScalingGroup, GroupMetrics, OnDemandAllocationStrategy, @@ -12,7 +13,7 @@ import { UpdatePolicy, } from "aws-cdk-lib/aws-autoscaling"; import { Metric } from "aws-cdk-lib/aws-cloudwatch"; -import type { IKeyPair, ISecurityGroup, IVpc } from "aws-cdk-lib/aws-ec2"; +import type { IKeyPair, IVpc, CfnInstance } from "aws-cdk-lib/aws-ec2"; import { MachineImage, OperatingSystemType, @@ -24,12 +25,11 @@ import { Instance, InstanceType, SecurityGroup, - CfnVolume, + Port, } from "aws-cdk-lib/aws-ec2"; import type { RoleProps } from "aws-cdk-lib/aws-iam"; import { ManagedPolicy, - Policy, PolicyDocument, PolicyStatement, Role, @@ -41,6 +41,7 @@ import { KeyPair } from "cdk-ec2-key-pair"; import { Construct } from "constructs"; import padStart from "lodash/padStart"; import range from "lodash/range"; +import { buildPercentScalingSteps } from "./internal/buildPercentScalingSteps"; import { cloudConfigBuild } from "./internal/cloudConfigBuild"; import { cloudConfigYamlDump } from "./internal/cloudConfigYamlDump"; import { namer } from "./internal/namer"; @@ -57,8 +58,9 @@ export interface CiStorageProps { /** Instance Profile Role inline policies to be used for all created * instances. */ inlinePolicies: RoleProps["inlinePolicies"]; - /** Id of the Security Group to set for the created instances. */ - securityGroupId: string; + /** All instance names (and hostname for the host instances) will be prefixed + * with that value, separated by "-". */ + instanceNamePrefix: string; /** A Hosted Zone to register the host instances in. */ hostedZone?: { /** Id of the Zone. */ @@ -83,6 +85,10 @@ export interface CiStorageProps { imageSsmName: string; /** Size of the root volume. */ volumeGb: number; + /** Size of swap file (if you need it). */ + swapSizeGb?: number; + /** If set, mounts /var/lib/docker to tmpfs with the provided max size. */ + tmpfsMaxSizeGb?: number; /** The list of requirements to choose Spot Instances. */ instanceRequirements: [ CfnAutoScalingGroup.InstanceRequirementsProperty, @@ -104,6 +110,8 @@ export interface CiStorageProps { periodSec: number; /** Value to use for the target percentage of active (busy) runners. */ value: number; + /** Desired number of ScalingInterval items in scalingSteps. */ + scalingSteps?: number; }; /** Minimal number of idle runners to keep, depending on the daytime. If * the auto scaling group has less than this number of instances, the new @@ -132,18 +140,22 @@ export interface CiStorageProps { * sparse-checkout that directory. The format is Dockerfile-compatible: * https://github.com/owner/repo[#[branch]:/directory/with/compose/] */ ghDockerComposeDirectoryUrl: string; + /** List of profiles from docker-compose to additionally start. */ + dockerComposeProfiles?: string[]; /** SSM parameter name which holds the reference to an instance image. */ imageSsmName: string; - /** IOPS of the docker volume. */ - volumeIops: number; - /** Throughput of the docker volume in MiB/s. */ - volumeThroughput: number; - /** Size of the docker volume. */ - volumeGb: number; + /** Size of swap file (if you need it). */ + swapSizeGb?: number; + /** If set, mounts /var/lib/docker to tmpfs with the provided max size and + * copies it from the old instance when the instance gets replaced. */ + tmpfsMaxSizeGb?: number; /** Full name of the Instance type. */ instanceType: string; /** Number of instances to create. */ machines: number; + /** Ports to be open in the security group for connection from all runners + * to the host. */ + ports: Array<{ port: Port; description: string }>; }; } @@ -171,24 +183,25 @@ export interface CiStorageProps { */ export class CiStorage extends Construct { public readonly vpc: IVpc; - public readonly securityGroup: ISecurityGroup; + public readonly securityGroup: SecurityGroup; public readonly keyPair: IKeyPair; public readonly keyPairPrivateKeySecretName: string; public readonly roles: { runner: Role; host: Role }; public readonly launchTemplate: LaunchTemplate; public readonly autoScalingGroup: AutoScalingGroup; public readonly hostedZone?: IHostedZone; - public readonly hostInstances: Instance[] = []; - public readonly hostVolumes: CfnVolume[] = []; + public readonly hosts: Array<{ + fqdn: string | undefined; + instance: Instance; + }> = []; constructor( public readonly scope: Construct, public readonly key: string, public readonly props: CiStorageProps, ) { - super(scope, key); - - const keyNamer = namer(key as any); + super(scope, namer(key as any).pascal); + const instanceNamePrefix = namer(props.instanceNamePrefix as any); this.vpc = props.vpc; @@ -224,33 +237,52 @@ export class CiStorage extends Construct { ], inlinePolicies: { ...props.inlinePolicies, - [namer(keyNamer, "key", "pair", "policy").pascal]: + [namer("key", "pair", "policy").pascal]: new PolicyDocument({ + statements: [ + new PolicyStatement({ + actions: ["secretsmanager:GetSecretValue"], + resources: [ + Stack.of(this).formatArn({ + service: "secretsmanager", + resource: "secret", + resourceName: `${this.keyPairPrivateKeySecretName}*`, + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + }), + ], + }), + ], + }), + [namer("gh", "token", "policy").pascal]: new PolicyDocument({ + statements: [ + new PolicyStatement({ + actions: ["secretsmanager:GetSecretValue"], + resources: [ + Stack.of(this).formatArn({ + service: "secretsmanager", + resource: "secret", + resourceName: `${props.ghTokenSecretName}*`, + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + }), + ], + }), + ], + }), + [namer("signal", "resource", "policy").pascal]: new PolicyDocument({ statements: [ new PolicyStatement({ - actions: ["secretsmanager:GetSecretValue"], - resources: [ - Stack.of(this).formatArn({ - service: "secretsmanager", - resource: "secret", - resourceName: `${this.keyPairPrivateKeySecretName}*`, - arnFormat: ArnFormat.COLON_RESOURCE_NAME, - }), - ], + actions: ["ec2:DescribeInstances"], + resources: ["*"], + // Describe* don't support resource-level permissions. }), - ], - }), - [namer(keyNamer, "gh", "token", "policy").pascal]: - new PolicyDocument({ - statements: [ new PolicyStatement({ - actions: ["secretsmanager:GetSecretValue"], + actions: ["cloudformation:SignalResource"], resources: [ Stack.of(this).formatArn({ - service: "secretsmanager", - resource: "secret", - resourceName: `${props.ghTokenSecretName}*`, - arnFormat: ArnFormat.COLON_RESOURCE_NAME, + service: "cloudformation", + resource: "stack", + resourceName: `${Stack.of(this).stackName}/*`, + arnFormat: ArnFormat.SLASH_RESOURCE_NAME, }), ], }), @@ -264,102 +296,21 @@ export class CiStorage extends Construct { { const id = namer("sg"); - this.securityGroup = SecurityGroup.fromSecurityGroupId( - this, - id.pascal, - props.securityGroupId, - ); - } - - { - const userData = UserData.custom( - cloudConfigYamlDump( - cloudConfigBuild({ - fqdn: "", - ghTokenSecretName: props.ghTokenSecretName, - ghDockerComposeDirectoryUrl: - props.runner.ghDockerComposeDirectoryUrl, - keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName, - timeZone: props.timeZone, - mount: undefined, - }), - ), - ); - const id = namer("launch", "template"); - this.launchTemplate = new LaunchTemplate(this, id.pascal, { - launchTemplateName: keyNamer.pathKebabFrom(scope), - machineImage: MachineImage.fromSsmParameter(props.runner.imageSsmName, { - os: OperatingSystemType.LINUX, - }), - keyPair: this.keyPair, - role: this.roles.runner, // LaunchTemplate creates InstanceProfile internally - blockDevices: [ - { - deviceName: "/dev/sda1", - volume: BlockDeviceVolume.ebs(props.runner.volumeGb, { - encrypted: true, - volumeType: EbsDeviceVolumeType.GP2, - deleteOnTermination: true, - }), - }, - ], - userData, - securityGroup: this.securityGroup, - requireImdsv2: true, - httpPutResponseHopLimit: 2, - }); - this.launchTemplate.node.addDependency(this.keyPair); - } - - { - const id = namer("auto", "scaling", "group"); - this.autoScalingGroup = new AutoScalingGroup(this, id.pascal, { - autoScalingGroupName: keyNamer.pathKebabFrom(scope), + this.securityGroup = new SecurityGroup(this, id.pascal, { + securityGroupName: id.pathKebabFrom(this), + description: id.pathKebabFrom(this), vpc: this.vpc, - maxCapacity: props.runner.scale.maxCapacity, - maxInstanceLifetime: props.runner.scale.maxInstanceLifetime, - mixedInstancesPolicy: { - instancesDistribution: { - onDemandAllocationStrategy: OnDemandAllocationStrategy.LOWEST_PRICE, - onDemandPercentageAboveBaseCapacity: - props.runner.scale.onDemandPercentageAboveBaseCapacity, - spotAllocationStrategy: - SpotAllocationStrategy.PRICE_CAPACITY_OPTIMIZED, - }, - launchTemplate: this.launchTemplate, - launchTemplateOverrides: props.runner.instanceRequirements.map( - (req) => ({ - instanceRequirements: req, - }), - ), - }, - cooldown: Duration.seconds(30), - defaultInstanceWarmup: Duration.seconds(60), - groupMetrics: [GroupMetrics.all()], - updatePolicy: UpdatePolicy.rollingUpdate(), }); - Tags.of(this.autoScalingGroup).add( - "Name", - namer(keyNamer, "runner").kebab, - ); - this.autoScalingGroup.scaleToTrackMetric("ActiveRunnersPercent", { - metric: new Metric({ - namespace: "ci-storage/metrics", - metricName: "ActiveRunnersPercent", - dimensionsMap: { GH_REPOSITORY: props.runner.ghRepository }, - period: Duration.seconds( - props.runner.scale.maxActiveRunnersPercent.periodSec, - ), - statistic: "max", - }), - targetValue: props.runner.scale.maxActiveRunnersPercent.value, - }); - for (const { id, value, cron } of props.runner.scale.minCapacity) { - this.autoScalingGroup.scaleOnSchedule(id, { - minCapacity: value, - timeZone: cron.timeZone ?? props.timeZone, - schedule: Schedule.cron(cron), - }); + Tags.of(this.securityGroup).add("Name", id.pathKebabFrom(this)); + for (const { port, description } of [ + { port: Port.tcp(22), description: "SSH" }, // to copy RAM drive from host to host + ...props.host.ports, + ]) { + this.securityGroup.addIngressRule( + this.securityGroup, + port, + `from runners and host to ${description}`, + ); } } @@ -381,36 +332,14 @@ export class CiStorage extends Construct { ); for (const i in range(props.host.machines)) { const id = namer( + instanceNamePrefix, "host", - namer(padStart(i + 1, 3, "0").toString() as any), + padStart(i + 1, 3, "0").toString() as Lowercase, ); - const recordName = namer(keyNamer, id).kebab; + const recordName = id.kebab; const fqdn = this.hostedZone ? recordName + "." + this.hostedZone.zoneName.replace(/\.$/, "") - : ""; - - // Unfortunately, there is no way in CDK to auto re-attach the volume to - // an instance if that instance gets replaced. This is because - // CloudFormation first launches a new instance while keeping the old - // instance still running, so the volume can't be attached to the new - // instance - it's already attached to the old one. The solution we use - // here is to do the volume attachment via cloud-config at the new - // instance's initial boot: it first stops the old instance from the new - // one ("aws ec2 stop-instances"), then detaches the volume, and then - // attaches it to the current instance. See logic in - // cloudConfigBuild.ts. - const volumeId = namer(id, "volume"); - const volume = new CfnVolume(this, volumeId.pascal, { - availabilityZone: this.vpc.availabilityZones[0], - autoEnableIo: true, - encrypted: true, - iops: props.host.volumeIops, - throughput: props.host.volumeThroughput, - size: props.host.volumeGb, - volumeType: "gp3", - }); - Tags.of(volume).add("Name", volumeId.pathKebabFrom(this)); - this.hostVolumes.push(volume); + : undefined; const userData = UserData.custom( cloudConfigYamlDump( @@ -419,16 +348,35 @@ export class CiStorage extends Construct { ghTokenSecretName: props.ghTokenSecretName, ghDockerComposeDirectoryUrl: props.host.ghDockerComposeDirectoryUrl, + dockerComposeEnv: {}, + dockerComposeProfiles: props.host.dockerComposeProfiles ?? [], keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName, timeZone: props.timeZone, - mount: { volumeId: volume.attrVolumeId, path: "/mnt" }, + tmpfs: props.host.tmpfsMaxSizeGb + ? { + path: "/var/lib/docker", + maxSizeGb: props.host.tmpfsMaxSizeGb, + } + : undefined, + swapSizeGb: props.host.swapSizeGb, }), ), ); const instance = new Instance( this, - namer(id, namer("instance")).pascal, + // As opposed to all other places, here we MUST prepend the instance + // construct id with the FULL scope (typically owning stack name) due + // to this bug: https://github.com/aws/aws-cdk/issues/22695 - the full + // instance id must be globally unique across all stacks, otherwise + // CDK fails to create automatic launch templates for them. + // + // With the current code, the auto-created launch template name is: + // - ..........Stk + Cnstrct + MyCiHost001 + Instance + LaunchTemplate + // + // But the instance id itself is ugly: + // - Cnstrct + Stk + Cnstrct + MyCiHost001 + Instance + namer(id, "instance").pathPascalFrom(this), { vpc: this.vpc, securityGroup: this.securityGroup, @@ -450,10 +398,13 @@ export class CiStorage extends Construct { ], userDataCausesReplacement: true, requireImdsv2: true, + detailedMonitoring: true, }, ); - Tags.of(instance.instance).add("Name", fqdn); - this.hostInstances.push(instance); + (instance.node.defaultChild as CfnInstance).cfnOptions.creationPolicy = + { resourceSignal: { count: 1, timeout: "PT15M" } }; + Tags.of(instance.instance).add("Name", fqdn ?? recordName); + this.hosts.push({ fqdn, instance }); if (this.hostedZone) { new ARecord(this, namer(id, namer("a")).pascal, { @@ -464,50 +415,121 @@ export class CiStorage extends Construct { }); } } + } - { - const id = namer("host", "volume", "policy"); - const conditions = { - StringEquals: { - ["ec2:ResourceTag/aws:cloudformation:stack-name"]: - Stack.of(this).stackName, - }, - }; - this.roles.host.attachInlinePolicy( - new Policy(this, id.pascal, { - policyName: namer(keyNamer, id).pascal, - statements: [ - new PolicyStatement({ - actions: ["ec2:DescribeVolumes", "ec2:DescribeInstances"], - resources: ["*"], - // Describe* don't support resource-level permissions and - // conditions. - }), - new PolicyStatement({ - actions: [ - "ec2:StopInstances", - "ec2:DetachVolume", - "ec2:AttachVolume", - ], - conditions, // filter by conditions, not by resource ARNs - resources: [ - Stack.of(this).formatArn({ - service: "ec2", - resource: "instance", - resourceName: "*", - arnFormat: ArnFormat.SLASH_RESOURCE_NAME, - }), - Stack.of(this).formatArn({ - service: "ec2", - resource: "volume", - resourceName: "*", - arnFormat: ArnFormat.SLASH_RESOURCE_NAME, - }), - ], - }), - ], + { + const userData = UserData.custom( + cloudConfigYamlDump( + cloudConfigBuild({ + fqdn: undefined, // no way to assign an unique hostname via LaunchTemplate + ghTokenSecretName: props.ghTokenSecretName, + ghDockerComposeDirectoryUrl: + props.runner.ghDockerComposeDirectoryUrl, + dockerComposeEnv: { + GH_REPOSITORY: props.runner.ghRepository, + GH_LABELS: `${instanceNamePrefix.kebab},ci-storage`, + // Future idea: each runner should know its host (for load + // balancing purposes); for now, we just hardcode the 1st one. + FORWARD_HOST: this.hosts[0].fqdn ?? "", + }, + dockerComposeProfiles: [], + keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName, + timeZone: props.timeZone, + tmpfs: props.runner.tmpfsMaxSizeGb + ? { + path: "/var/lib/docker", + maxSizeGb: props.runner.tmpfsMaxSizeGb, + } + : undefined, + swapSizeGb: props.runner.swapSizeGb, }), - ); + ), + ); + const id = namer("lt"); + this.launchTemplate = new LaunchTemplate(this, id.pascal, { + launchTemplateName: id.pathKebabFrom(this), + machineImage: MachineImage.fromSsmParameter(props.runner.imageSsmName, { + os: OperatingSystemType.LINUX, + }), + keyPair: this.keyPair, + role: this.roles.runner, // LaunchTemplate creates InstanceProfile internally + blockDevices: [ + { + deviceName: "/dev/sda1", + volume: BlockDeviceVolume.ebs(props.runner.volumeGb, { + encrypted: true, + volumeType: EbsDeviceVolumeType.GP2, + deleteOnTermination: true, + }), + }, + ], + userData, + securityGroup: this.securityGroup, + requireImdsv2: true, + httpPutResponseHopLimit: 2, + detailedMonitoring: true, + }); + this.launchTemplate.node.addDependency(this.keyPair); + } + + { + const id = namer("asg", "runner"); + this.autoScalingGroup = new AutoScalingGroup(this, id.pascal, { + autoScalingGroupName: id.pathKebabFrom(this), + vpc: this.vpc, + maxCapacity: props.runner.scale.maxCapacity, + maxInstanceLifetime: props.runner.scale.maxInstanceLifetime, + mixedInstancesPolicy: { + instancesDistribution: { + onDemandAllocationStrategy: OnDemandAllocationStrategy.LOWEST_PRICE, + onDemandPercentageAboveBaseCapacity: + props.runner.scale.onDemandPercentageAboveBaseCapacity, + spotAllocationStrategy: + SpotAllocationStrategy.PRICE_CAPACITY_OPTIMIZED, + }, + launchTemplate: this.launchTemplate, + launchTemplateOverrides: props.runner.instanceRequirements.map( + (req) => ({ + instanceRequirements: req, + }), + ), + }, + cooldown: Duration.seconds(30), + defaultInstanceWarmup: Duration.seconds(60), + groupMetrics: [GroupMetrics.all()], + updatePolicy: UpdatePolicy.rollingUpdate(), + }); + Tags.of(this.autoScalingGroup).add( + "Name", + namer(instanceNamePrefix, "runner").kebab, + ); + + const metric = new Metric({ + namespace: "ci-storage/metrics", + metricName: "ActiveRunnersPercent", + dimensionsMap: { GH_REPOSITORY: props.runner.ghRepository }, + period: Duration.seconds( + props.runner.scale.maxActiveRunnersPercent.periodSec, + ), + statistic: "max", + }); + const scalingSteps = buildPercentScalingSteps( + props.runner.scale.maxActiveRunnersPercent.value, + props.runner.scale.maxActiveRunnersPercent.scalingSteps ?? 6, + ); + this.autoScalingGroup.scaleOnMetric("ActiveRunnersPercent", { + metric, + adjustmentType: AdjustmentType.PERCENT_CHANGE_IN_CAPACITY, + scalingSteps, + evaluationPeriods: 1, + datapointsToAlarm: 1, + }); + for (const { id, value, cron } of props.runner.scale.minCapacity) { + this.autoScalingGroup.scaleOnSchedule(id, { + minCapacity: value, + timeZone: cron.timeZone ?? props.timeZone, + schedule: Schedule.cron({ minute: "0", ...cron }), + }); } } } diff --git a/src/__tests__/CiStorage.test.ts b/src/__tests__/CiStorage.test.ts index 9b83038..2bbcee1 100644 --- a/src/__tests__/CiStorage.test.ts +++ b/src/__tests__/CiStorage.test.ts @@ -1,6 +1,6 @@ import { App, Duration, Stack } from "aws-cdk-lib"; import { Template } from "aws-cdk-lib/assertions"; -import { Vpc } from "aws-cdk-lib/aws-ec2"; +import { Port, Vpc } from "aws-cdk-lib/aws-ec2"; import type { Construct } from "constructs"; import { CiStorage } from "../CiStorage"; import { namer } from "../internal/namer"; @@ -20,10 +20,10 @@ class CiStorageStack extends Stack { }); this.vpc = new Vpc(this, "Vpc", {}); - this.ciStorage = new CiStorage(this, "CiStorage", { + this.ciStorage = new CiStorage(this, "cnstrct", { vpc: this.vpc, inlinePolicies: {}, - securityGroupId: "test-securityGroupId", + instanceNamePrefix: "my-ci", hostedZone: { hostedZoneId: "test-hostedZoneId", zoneName: "test-zoneName", @@ -36,6 +36,8 @@ class CiStorageStack extends Stack { "https://github.com/dimikot/ci-storage#:docker", imageSsmName: "test-imageSsmName", volumeGb: 50, + swapSizeGb: 8, + tmpfsMaxSizeGb: 4, instanceRequirements: [ { memoryMiB: { min: 8192, max: 16384 }, @@ -47,6 +49,7 @@ class CiStorageStack extends Stack { maxActiveRunnersPercent: { periodSec: 600, value: 70, + scalingSteps: 10, }, minCapacity: [ { @@ -67,12 +70,15 @@ class CiStorageStack extends Stack { host: { ghDockerComposeDirectoryUrl: "https://github.com/dimikot/ci-storage#:docker", + dockerComposeProfiles: ["ci"], imageSsmName: "test-imageSsmName", - volumeIops: 3000, - volumeThroughput: 125, - volumeGb: 200, + tmpfsMaxSizeGb: 4, instanceType: "t3.large", machines: 1, + ports: [ + { port: Port.tcp(10022), description: "ci-storage container" }, + { port: Port.tcpRange(42000, 42042), description: "test ports" }, + ], }, }); } @@ -80,6 +86,6 @@ class CiStorageStack extends Stack { test("CiStorage", () => { const app = new App(); - const stack = new CiStorageStack(app, namer("test"), {}); + const stack = new CiStorageStack(app, namer("stk"), {}); expect(Template.fromStack(stack).toJSON()).toMatchSnapshot(); }); diff --git a/src/__tests__/__snapshots__/CiStorage.test.ts.snap b/src/__tests__/__snapshots__/CiStorage.test.ts.snap index b7060dd..1a6b80e 100644 --- a/src/__tests__/__snapshots__/CiStorage.test.ts.snap +++ b/src/__tests__/__snapshots__/CiStorage.test.ts.snap @@ -15,9 +15,9 @@ exports[`CiStorage 1`] = ` }, }, "Resources": { - "CiStorageAutoScalingGroupASGFCC25A25": { + "CnstrctAsgRunnerASG489B3A08": { "Properties": { - "AutoScalingGroupName": "test-cistorage", + "AutoScalingGroupName": "stk-cnstrct-asgrunner", "Cooldown": "30", "DefaultInstanceWarmup": 60, "MaxInstanceLifetime": 86400, @@ -37,11 +37,11 @@ exports[`CiStorage 1`] = ` "LaunchTemplate": { "LaunchTemplateSpecification": { "LaunchTemplateId": { - "Ref": "CiStorageLaunchTemplate73370FC5", + "Ref": "CnstrctLtAEEE5196", }, "Version": { "Fn::GetAtt": [ - "CiStorageLaunchTemplate73370FC5", + "CnstrctLtAEEE5196", "LatestVersionNumber", ], }, @@ -66,7 +66,7 @@ exports[`CiStorage 1`] = ` { "Key": "Name", "PropagateAtLaunch": true, - "Value": "ci-storage-runner", + "Value": "my-ci-runner", }, ], "VPCZoneIdentifier": [ @@ -95,134 +95,450 @@ exports[`CiStorage 1`] = ` }, }, }, - "CiStorageAutoScalingGroupScalingPolicyActiveRunnersPercent7591E33E": { + "CnstrctAsgRunnerActiveRunnersPercentLowerAlarm3D348E7B": { "Properties": { + "AlarmActions": [ + { + "Ref": "CnstrctAsgRunnerActiveRunnersPercentLowerPolicy9E822646", + }, + ], + "AlarmDescription": "Lower threshold scaling alarm", + "ComparisonOperator": "LessThanOrEqualToThreshold", + "DatapointsToAlarm": 1, + "Dimensions": [ + { + "Name": "GH_REPOSITORY", + "Value": "time-loop/slapdash", + }, + ], + "EvaluationPeriods": 1, + "MetricName": "ActiveRunnersPercent", + "Namespace": "ci-storage/metrics", + "Period": 600, + "Statistic": "Maximum", + "Tags": [ + { + "Key": "Name", + "Value": "my-ci-runner", + }, + ], + "Threshold": 63, + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "CnstrctAsgRunnerActiveRunnersPercentLowerPolicy9E822646": { + "Properties": { + "AdjustmentType": "PercentChangeInCapacity", "AutoScalingGroupName": { - "Ref": "CiStorageAutoScalingGroupASGFCC25A25", + "Ref": "CnstrctAsgRunnerASG489B3A08", }, - "PolicyType": "TargetTrackingScaling", - "TargetTrackingConfiguration": { - "CustomizedMetricSpecification": { - "Dimensions": [ - { - "Name": "GH_REPOSITORY", - "Value": "time-loop/slapdash", - }, - ], - "MetricName": "ActiveRunnersPercent", - "Namespace": "ci-storage/metrics", - "Statistic": "Maximum", + "MetricAggregationType": "Maximum", + "PolicyType": "StepScaling", + "StepAdjustments": [ + { + "MetricIntervalLowerBound": -7, + "MetricIntervalUpperBound": 0, + "ScalingAdjustment": -10, + }, + { + "MetricIntervalLowerBound": -14, + "MetricIntervalUpperBound": -7, + "ScalingAdjustment": -20, + }, + { + "MetricIntervalLowerBound": -21, + "MetricIntervalUpperBound": -14, + "ScalingAdjustment": -30, + }, + { + "MetricIntervalLowerBound": -28, + "MetricIntervalUpperBound": -21, + "ScalingAdjustment": -40, + }, + { + "MetricIntervalLowerBound": -35, + "MetricIntervalUpperBound": -28, + "ScalingAdjustment": -50, + }, + { + "MetricIntervalLowerBound": -42, + "MetricIntervalUpperBound": -35, + "ScalingAdjustment": -60, + }, + { + "MetricIntervalLowerBound": -49, + "MetricIntervalUpperBound": -42, + "ScalingAdjustment": -70, + }, + { + "MetricIntervalLowerBound": -56, + "MetricIntervalUpperBound": -49, + "ScalingAdjustment": -80, + }, + { + "MetricIntervalUpperBound": -56, + "ScalingAdjustment": -90, + }, + ], + }, + "Type": "AWS::AutoScaling::ScalingPolicy", + }, + "CnstrctAsgRunnerActiveRunnersPercentUpperAlarm9D4818B9": { + "Properties": { + "AlarmActions": [ + { + "Ref": "CnstrctAsgRunnerActiveRunnersPercentUpperPolicy07F985DA", + }, + ], + "AlarmDescription": "Upper threshold scaling alarm", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "DatapointsToAlarm": 1, + "Dimensions": [ + { + "Name": "GH_REPOSITORY", + "Value": "time-loop/slapdash", }, - "TargetValue": 70, + ], + "EvaluationPeriods": 1, + "MetricName": "ActiveRunnersPercent", + "Namespace": "ci-storage/metrics", + "Period": 600, + "Statistic": "Maximum", + "Tags": [ + { + "Key": "Name", + "Value": "my-ci-runner", + }, + ], + "Threshold": 70, + }, + "Type": "AWS::CloudWatch::Alarm", + }, + "CnstrctAsgRunnerActiveRunnersPercentUpperPolicy07F985DA": { + "Properties": { + "AdjustmentType": "PercentChangeInCapacity", + "AutoScalingGroupName": { + "Ref": "CnstrctAsgRunnerASG489B3A08", }, + "MetricAggregationType": "Maximum", + "PolicyType": "StepScaling", + "StepAdjustments": [ + { + "MetricIntervalLowerBound": 0, + "MetricIntervalUpperBound": 3, + "ScalingAdjustment": 4, + }, + { + "MetricIntervalLowerBound": 3, + "MetricIntervalUpperBound": 6, + "ScalingAdjustment": 9, + }, + { + "MetricIntervalLowerBound": 6, + "MetricIntervalUpperBound": 9, + "ScalingAdjustment": 13, + }, + { + "MetricIntervalLowerBound": 9, + "MetricIntervalUpperBound": 12, + "ScalingAdjustment": 17, + }, + { + "MetricIntervalLowerBound": 12, + "MetricIntervalUpperBound": 15, + "ScalingAdjustment": 21, + }, + { + "MetricIntervalLowerBound": 15, + "MetricIntervalUpperBound": 18, + "ScalingAdjustment": 26, + }, + { + "MetricIntervalLowerBound": 18, + "MetricIntervalUpperBound": 21, + "ScalingAdjustment": 30, + }, + { + "MetricIntervalLowerBound": 21, + "MetricIntervalUpperBound": 24, + "ScalingAdjustment": 34, + }, + { + "MetricIntervalLowerBound": 24, + "MetricIntervalUpperBound": 27, + "ScalingAdjustment": 39, + }, + { + "MetricIntervalLowerBound": 27, + "ScalingAdjustment": 43, + }, + ], }, "Type": "AWS::AutoScaling::ScalingPolicy", }, - "CiStorageAutoScalingGroupScheduledActionCaWorkDayEnds58BA31B2": { + "CnstrctAsgRunnerScheduledActionCaWorkDayEnds81158DAD": { "Properties": { "AutoScalingGroupName": { - "Ref": "CiStorageAutoScalingGroupASGFCC25A25", + "Ref": "CnstrctAsgRunnerASG489B3A08", }, "MinSize": 5, - "Recurrence": "* 18 * * *", + "Recurrence": "0 18 * * *", "TimeZone": "America/Los_Angeles", }, "Type": "AWS::AutoScaling::ScheduledAction", }, - "CiStorageAutoScalingGroupScheduledActionCaWorkDayStartsAAC7A1B8": { + "CnstrctAsgRunnerScheduledActionCaWorkDayStarts6A1CC23B": { "Properties": { "AutoScalingGroupName": { - "Ref": "CiStorageAutoScalingGroupASGFCC25A25", + "Ref": "CnstrctAsgRunnerASG489B3A08", }, "MinSize": 10, - "Recurrence": "* 8 * * *", + "Recurrence": "0 8 * * *", "TimeZone": "America/Los_Angeles", }, "Type": "AWS::AutoScaling::ScheduledAction", }, - "CiStorageHost001A90FC20D0": { + "CnstrctHostRole5DD9F366": { "Properties": { - "HostedZoneId": "test-hostedZoneId", - "Name": "ci-storage-host-001.test-zoneName.", - "ResourceRecords": [ + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ { - "Fn::GetAtt": [ - "CiStorageHost001Instance74416F6B4260e81d2d555257", - "PrivateIp", + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AmazonEC2RoleforSSM", + ], + ], + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/CloudWatchAgentServerPolicy", + ], ], }, ], - "TTL": "60", - "Type": "A", + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": "secretsmanager:GetSecretValue", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":secretsmanager:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":secret:ec2-ssh-key/", + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "KeyPairPolicy", + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "secretsmanager:GetSecretValue", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":secretsmanager:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":secret:ci-storage/gh-token*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "GhTokenPolicy", + }, + { + "PolicyDocument": { + "Statement": [ + { + "Action": "ec2:DescribeInstances", + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": "cloudformation:SignalResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":cloudformation:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stack/Stk/*", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "SignalResourcePolicy", + }, + ], + "RoleName": "StkCnstrctHostRole", }, - "Type": "AWS::Route53::RecordSet", + "Type": "AWS::IAM::Role", }, - "CiStorageHost001Instance74416F6B4260e81d2d555257": { + "CnstrctLtAEEE5196": { "DependsOn": [ - "CiStorageHostRole9FF47CEB", + "CnstrctRunnerRole341E54FA", ], "Properties": { - "AvailabilityZone": { - "Fn::Select": [ - 0, + "LaunchTemplateData": { + "BlockDeviceMappings": [ { - "Fn::GetAZs": "", + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "Encrypted": true, + "VolumeSize": 50, + "VolumeType": "gp2", + }, }, ], - }, - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "Encrypted": true, - "VolumeSize": 20, - "VolumeType": "gp2", + "IamInstanceProfile": { + "Arn": { + "Fn::GetAtt": [ + "CnstrctLtProfileE3059BA8", + "Arn", + ], }, }, - ], - "IamInstanceProfile": { - "Ref": "CiStorageHost001InstanceInstanceProfileAF3BBAE6", - }, - "ImageId": { - "Ref": "SsmParameterValuetestimageSsmNameC96584B6F00A464EAD1953AFF4B05118Parameter", - }, - "InstanceType": "t3.large", - "KeyName": { - "Fn::GetAtt": [ - "CiStorageSshIdRsaEC2KeyPairtestcistoragesshidrsa26A4353E", - "KeyPairName", - ], - }, - "LaunchTemplate": { - "LaunchTemplateName": "Host001InstanceLaunchTemplate", - "Version": { + "ImageId": { + "Ref": "SsmParameterValuetestimageSsmNameC96584B6F00A464EAD1953AFF4B05118Parameter", + }, + "KeyName": { "Fn::GetAtt": [ - "CiStorageHost001InstanceLaunchTemplate75DE93DE", - "LatestVersionNumber", + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", ], }, - }, - "SecurityGroupIds": [ - "test-securityGroupId", - ], - "SubnetId": { - "Ref": "VpcPrivateSubnet1Subnet536B997A", - }, - "Tags": [ - { - "Key": "Name", - "Value": "ci-storage-host-001.test-zoneName", + "MetadataOptions": { + "HttpPutResponseHopLimit": 2, + "HttpTokens": "required", }, - ], - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "#cloud-config + "Monitoring": { + "Enabled": true, + }, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/Lt", + }, + ], + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/Lt", + }, + ], + }, + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#cloud-config timezone: America/Los_Angeles -fqdn: ci-storage-host-001.test-zoneName +swap: + filename: /var/swapfile + size: auto + maxsize: 8589934592 +mounts: + - - tmpfs + - /var/lib/docker + - tmpfs + - defaults,noatime,exec,mode=0710,nr_inodes=0,size=4G + - "0" + - "0" apt_sources: - source: deb https://cli.github.com/packages stable main keyid: 23F3D4EA75716059 @@ -237,6 +553,9 @@ packages: - docker-ce-cli - containerd.io - docker-compose-plugin + - qemu + - qemu-user-static + - binfmt-support - git - gosu - mc @@ -244,139 +563,104 @@ packages: - apt-transport-https - ca-certificates - tzdata + - atop + - iotop + - htop + - bwm-ng + - jq write_files: - path: /etc/sysctl.d/enable-ipv4-forwarding.conf content: | net.ipv4.conf.all.forwarding=1 - - path: /etc/sysctl.d/lower-fs-inodes-eviction-from-cache.conf + - path: /etc/default/atop + content: | + LOGOPTS="-R" + LOGINTERVAL=15 + LOGGENERATIONS=4 + - path: /etc/environment + append: true + content: | + TZ="America/Los_Angeles" + - path: /etc/environment + append: true + content: | + LESS="RS" + - path: /etc/docker/daemon.json + permissions: "0644" content: | - vm.vfs_cache_pressure=0 - vm.swappiness=10 - - path: /var/lib/cloud/scripts/per-once/define-tz-env.sh + { + "log-driver": "syslog", + "log-opts": { + "tag": "docker/{{.Name}}" + }, + "runtimes": { + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" + } + }, + "default-runtime": "sysbox-runc", + "userns-remap": "sysbox" + } + - path: /var/lib/cloud/scripts/per-once/apply-services-configs.sh permissions: "0755" content: | #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace - - echo 'TZ="America/Los_Angeles"' >> /etc/environment + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + service atop restart || true + sysctl --system - path: /var/lib/cloud/scripts/per-once/increase-docker-shutdown-timeout.sh permissions: "0755" content: | #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace - + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") sed -i -E '/TimeoutStartSec=.*/a TimeoutStopSec=3600' /usr/lib/systemd/system/docker.service systemctl daemon-reload + - path: /var/lib/cloud/scripts/per-once/add-ubuntu-user-to-docker-group-to-access-socket.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -aG docker ubuntu + - path: /var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + systemctl stop docker docker.socket || true + wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" + dpkg -i /tmp/sysbox-ce.deb + rm -f /tmp/sysbox-ce.deb - path: /var/lib/cloud/scripts/per-once/switch-ssm-user-to-ubuntu-on-login.sh permissions: "0755" content: | #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace - + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") echo '[ "$0$@" = "sh" ] && ENV= sudo -u ubuntu -i' > /etc/profile.ssm-user mkdir -p /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/ ( echo '[Service]' echo 'Environment="ENV=/etc/profile.ssm-user"' - ) > /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/sh-env.conf - systemctl daemon-reload - systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true - - path: /var/lib/cloud/scripts/per-once/detach-volume-from-old-instance-and-mount.sh - permissions: "0755" - content: | - #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace - - export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - volume_id="", - { - "Fn::GetAtt": [ - "CiStorageHost001Volume0C0FD514", - "VolumeId", - ], - }, - "" - volume_hash="\${volume_id##vol-}" - volume_dir="/mnt" - volume_label="MNT" - instance_id=$(ec2metadata --instance-id) - - # Stop the old instances. This causes a small downtime of the host - # service, but it's acceptable for the CI use case. - old_instance_id=$( - aws ec2 describe-volumes \\ - --volume-ids "$volume_id" \\ - --query "Volumes[].Attachments[].InstanceId" \\ - --output text - ) - if [[ "$old_instance_id" != "" ]]; then - sent_command=0 - while ! aws ec2 describe-instances \\ - --instance-ids "$old_instance_id" \\ - --query "Reservations[].Instances[].State.Name" \\ - --output text \\ - | egrep -q "stopped|terminated" - do - if [[ "$sent_command" == "0" ]]; then - sent_command=1 - aws ec2 stop-instances --instance-ids "$old_instance_id" || true - fi - sleep 1 - done - fi - - # Detach volume from the old instance. - sent_command=0 - while ! aws ec2 describe-volumes \\ - --volume-ids "$volume_id" \\ - --query "Volumes[].State" \\ - --output text \\ - | grep -q available - do - if [[ "$sent_command" == "0" ]]; then - sent_command=1 - aws ec2 detach-volume --volume-id "$volume_id" --force || true - fi - sleep 0.2; - done - - # Attach volume to this instance and wait for the device to appear. - sent_command=0 - while ! ls /dev/disk/by-id | grep -q "$volume_hash"; do - if [[ "$sent_command" == "0" ]]; then - sent_command=1 - aws ec2 attach-volume --volume-id "$volume_id" --instance-id "$instance_id" --device /dev/sdf - fi - sleep 0.2 - done - - # Mount volume if it already exists, or create the filesystem. - lsblk - ls -la /dev/disk/by-id - device=$(echo /dev/disk/by-id/*$volume_hash) - if ! grep -q "LABEL=$volume_label" /etc/fstab; then - echo "LABEL=$volume_label $volume_dir auto defaults,noatime,data=writeback 0 0" >> /etc/fstab - fi - mount -a || true - if ! mountpoint "$volume_dir"; then - mkfs -t ext4 "$device" - tune2fs -L "$volume_label" "$device" - mount -a - systemctl stop docker docker.socket - ls -la /var/lib/docker - cp -axT /var/lib/docker "$volume_dir/var_lib_docker" - mv -f /var/lib/docker /var/lib/docker.old - ln -sT "$volume_dir/var_lib_docker" /var/lib/docker - systemctl start docker docker.socket - fi - ls -la "$volume_dir" + ) > /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/sh-env.conf + systemctl daemon-reload + systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true + - path: /etc/rsyslog.d/01-docker-tag-to-serial-console.conf + permissions: "0644" + content: | + if $syslogtag startswith 'docker/' then -/dev/console + # It will also write to /var/log/syslog as usual. + - path: /var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -a -G tty syslog + systemctl restart rsyslog - path: /var/lib/cloud/scripts/per-boot/run-docker-compose-on-boot.sh permissions: "0755" content: | #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace - - echo "*/1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh 2>&1 | logger -t run-docker-compose" > /etc/cron.d/run-docker-compose + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + echo "*/2 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-logs 2>&1 | logger -t docker/run-docker-compose" > /etc/cron.d/run-docker-compose exec /home/ubuntu/run-docker-compose.sh - path: /home/ubuntu/run-docker-compose.sh owner: ubuntu:ubuntu @@ -387,38 +671,49 @@ write_files: # Switch to non-privileged user if running as root. if [[ $(whoami) != "ubuntu" ]]; then - exec gosu ubuntu:ubuntu "$BASH_SOURCE" + exec gosu ubuntu "$BASH_SOURCE" "$@" fi # Ensure there is only one instance of this script running. exec {FD}<$BASH_SOURCE flock -n "$FD" || { echo "Already running."; exit 0; } - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + + # Make sure we're using the right timezone; it may be not up + # to date in the current environment during the very 1st run + # from run-docker-compose-on-boot.sh. + source /etc/environment + export TZ # Load private and public keys from Secrets Manager to ~/.ssh. - export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") mkdir -p ~/.ssh && chmod 700 ~/.ssh aws secretsmanager get-secret-value \\ --secret-id "ec2-ssh-key/", - { - "Fn::GetAtt": [ - "CiStorageSshIdRsaEC2KeyPairtestcistoragesshidrsa26A4353E", - "KeyPairName", - ], - }, - "/private" \\ + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private" \\ --query SecretString --output text \\ > ~/.ssh/ci-storage chmod 600 ~/.ssh/ci-storage ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub - # Load GitHub PAT from Secrets Manager and login to GitHub. + # Load GitHub PAT from Secrets Manager and log in to GitHub. aws secretsmanager get-secret-value \\ --secret-id "ci-storage/gh-token" \\ --query SecretString --output text \\ | gh auth login --with-token gh auth setup-git + # Log in to ghcr.io every hour. + config=~/.docker/config.json + if [[ ! -f $config ]] || find "$config" -type f -mmin +60 | grep -q .; then + gh auth token | docker login ghcr.io -u "$(gh api user -q .login)" --password-stdin + fi + # Pull the repository. mkdir -p ~/git && cd ~/git if [[ ! -d .git ]]; then @@ -431,80 +726,109 @@ write_files: git pull --rebase fi - # Run docker compose. - sudo usermod -aG docker ubuntu - export GH_TOKEN + # Process some tokens and print rate limits without xtrace. set +o xtrace GH_TOKEN=$(gh auth token) + echo "Docker Hub Rate Limits:" + docker_hub_token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token || true) + curl -s --head -H "Authorization: Bearer $docker_hub_token" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit || true + echo "GitHub Core Rate Limits:" + gh api -i -X HEAD /rate_limit | grep Ratelimit set -o xtrace - exec sg docker -c 'cd "docker" && docker compose pull && exec docker compose up --build -d' + + # Export env vars for docker compose. + export GH_TOKEN + export GH_REPOSITORY="time-loop/slapdash" + export GH_LABELS="my-ci,ci-storage" + export FORWARD_HOST="my-ci-host-001.test-zoneName" + + # It it's the very 1st run, start Docker service. We do not start it every run, + # because otherwise we wouldn't be able to "systemctl stop docker docker.socket" + # manually or while copying files from the old host. + file=~/.docker-started-after-first-git-clone + if [[ ! -f $file ]]; then + sudo systemctl start docker docker.socket + touch $file + fi + + # Run docker compose. + cd "docker" + docker pull ghcr.io/dimikot/ci-storage:main || true + docker pull ghcr.io/dimikot/ci-runner:main || true + docker compose up --build --remove-orphans -d + sleep 5 + if [[ "$1" != "--no-logs" ]]; then + docker compose logs -n 10 + fi + docker system prune --volumes -f - path: /home/ubuntu/.bash_profile owner: ubuntu:ubuntu permissions: "0644" defer: true content: | #!/bin/bash - if [ -d ~/git/"docker" ]; then + C_CMD="\\033[0;36m" + C_NO="\\033[0m" + if [[ -d ~/git/"docker" ]]; then cd ~/git/"docker" - echo '$ docker compose ps' + echo -e "$C_CMD\\$ docker compose ps$C_NO" docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" - echo + services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) + if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then + cmd="docker compose exec $services bash -l" + echo -e "$C_CMD\\$ $cmd$C_NO" + eval "$cmd" + fi fi ", + ], ], - ], + }, }, }, + "LaunchTemplateName": "stk-cnstrct-lt", + "TagSpecifications": [ + { + "ResourceType": "launch-template", + "Tags": [ + { + "Key": "Name", + "Value": "Stk/Cnstrct/Lt", + }, + ], + }, + ], }, - "Type": "AWS::EC2::Instance", + "Type": "AWS::EC2::LaunchTemplate", }, - "CiStorageHost001InstanceInstanceProfileAF3BBAE6": { + "CnstrctLtProfileE3059BA8": { "Properties": { "Roles": [ { - "Ref": "CiStorageHostRole9FF47CEB", + "Ref": "CnstrctRunnerRole341E54FA", }, ], }, "Type": "AWS::IAM::InstanceProfile", }, - "CiStorageHost001InstanceLaunchTemplate75DE93DE": { - "Properties": { - "LaunchTemplateData": { - "MetadataOptions": { - "HttpTokens": "required", - }, - }, - "LaunchTemplateName": "Host001InstanceLaunchTemplate", - }, - "Type": "AWS::EC2::LaunchTemplate", - }, - "CiStorageHost001Volume0C0FD514": { + "CnstrctMyCiHost001A9DE30324": { "Properties": { - "AutoEnableIO": true, - "AvailabilityZone": { - "Fn::Select": [ - 0, - { - "Fn::GetAZs": "", - }, - ], - }, - "Encrypted": true, - "Iops": 3000, - "Size": 200, - "Tags": [ + "HostedZoneId": "test-hostedZoneId", + "Name": "my-ci-host-001.test-zoneName.", + "ResourceRecords": [ { - "Key": "Name", - "Value": "test-cistorage-host001volume", + "Fn::GetAtt": [ + "CnstrctStkCnstrctMyCiHost001InstanceCCA3759A90e3e9f8dfbab5f0", + "PrivateIp", + ], }, ], - "Throughput": 125, - "VolumeType": "gp3", + "TTL": "60", + "Type": "A", }, - "Type": "AWS::EC2::Volume", + "Type": "AWS::Route53::RecordSet", }, - "CiStorageHostRole9FF47CEB": { + "CnstrctRunnerRole341E54FA": { "Properties": { "AssumeRolePolicyDocument": { "Statement": [ @@ -570,7 +894,7 @@ write_files: ":secret:ec2-ssh-key/", { "Fn::GetAtt": [ - "CiStorageSshIdRsaEC2KeyPairtestcistoragesshidrsa26A4353E", + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", "KeyPairName", ], }, @@ -582,7 +906,7 @@ write_files: ], "Version": "2012-10-17", }, - "PolicyName": "CiStorageKeyPairPolicy", + "PolicyName": "KeyPairPolicy", }, { "PolicyDocument": { @@ -614,160 +938,249 @@ write_files: ], "Version": "2012-10-17", }, - "PolicyName": "CiStorageGhTokenPolicy", + "PolicyName": "GhTokenPolicy", }, - ], - "RoleName": "TestCiStorageHostRole", - }, - "Type": "AWS::IAM::Role", - }, - "CiStorageHostVolumePolicy81A8AB83": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "ec2:DescribeVolumes", - "ec2:DescribeInstances", - ], - "Effect": "Allow", - "Resource": "*", - }, - { - "Action": [ - "ec2:StopInstances", - "ec2:DetachVolume", - "ec2:AttachVolume", - ], - "Condition": { - "StringEquals": { - "ec2:ResourceTag/aws:cloudformation:stack-name": "Test", - }, - }, - "Effect": "Allow", - "Resource": [ + { + "PolicyDocument": { + "Statement": [ { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":ec2:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":instance/*", - ], - ], + "Action": "ec2:DescribeInstances", + "Effect": "Allow", + "Resource": "*", }, { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":ec2:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":volume/*", + "Action": "cloudformation:SignalResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":cloudformation:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stack/Stk/*", + ], ], - ], + }, }, ], + "Version": "2012-10-17", }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CiStorageHostVolumePolicy", - "Roles": [ + "PolicyName": "SignalResourcePolicy", + }, + ], + "RoleName": "StkCnstrctRunnerRole", + }, + "Type": "AWS::IAM::Role", + }, + "CnstrctSgF5C70BA4": { + "Properties": { + "GroupDescription": "stk-cnstrct-sg", + "GroupName": "stk-cnstrct-sg", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1", + }, + ], + "Tags": [ { - "Ref": "CiStorageHostRole9FF47CEB", + "Key": "Name", + "Value": "stk-cnstrct-sg", }, ], + "VpcId": { + "Ref": "Vpc8378EB38", + }, + }, + "Type": "AWS::EC2::SecurityGroup", + }, + "CnstrctSgfromStkCnstrctSgCE9C2C74100223764E466": { + "Properties": { + "Description": "from runners and host to ci-storage container", + "FromPort": 10022, + "GroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "ToPort": 10022, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "CnstrctSgfromStkCnstrctSgCE9C2C74226B452137": { + "Properties": { + "Description": "from runners and host to SSH", + "FromPort": 22, + "GroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "ToPort": 22, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "CnstrctSgfromStkCnstrctSgCE9C2C744200042042B0BB20A9": { + "Properties": { + "Description": "from runners and host to test ports", + "FromPort": 42000, + "GroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + "ToPort": 42042, + }, + "Type": "AWS::EC2::SecurityGroupIngress", + }, + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69": { + "DeletionPolicy": "Delete", + "Properties": { + "Description": "Used to access ci-storage host from self-hosted runner nodes.", + "ExposePublicKey": false, + "KmsPrivate": "alias/aws/secretsmanager", + "KmsPublic": "alias/aws/secretsmanager", + "Name": "stk-cnstrct-sshidrsa", + "PublicKey": "", + "PublicKeyFormat": "OPENSSH", + "RemoveKeySecretsAfterDays": 0, + "SecretPrefix": "ec2-ssh-key/", + "ServiceToken": { + "Fn::GetAtt": [ + "EC2KeyNameManagerLambdaBE629145", + "Arn", + ], + }, + "StackName": "Stk", + "StorePublicKey": false, + "Tags": { + "CreatedByCfnCustomResource": "CFN::Resource::Custom::EC2-Key-Pair", + }, }, - "Type": "AWS::IAM::Policy", + "Type": "Custom::EC2-Key-Pair", + "UpdateReplacePolicy": "Delete", }, - "CiStorageLaunchTemplate73370FC5": { + "CnstrctStkCnstrctMyCiHost001InstanceCCA3759A90e3e9f8dfbab5f0": { + "CreationPolicy": { + "ResourceSignal": { + "Count": 1, + "Timeout": "PT15M", + }, + }, "DependsOn": [ - "CiStorageRunnerRole9B60440E", + "CnstrctHostRole5DD9F366", ], "Properties": { - "LaunchTemplateData": { - "BlockDeviceMappings": [ + "AvailabilityZone": { + "Fn::Select": [ + 0, { - "DeviceName": "/dev/sda1", - "Ebs": { - "DeleteOnTermination": true, - "Encrypted": true, - "VolumeSize": 50, - "VolumeType": "gp2", - }, + "Fn::GetAZs": "", }, ], - "IamInstanceProfile": { - "Arn": { - "Fn::GetAtt": [ - "CiStorageLaunchTemplateProfile1CF26B91", - "Arn", - ], + }, + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "DeleteOnTermination": true, + "Encrypted": true, + "VolumeSize": 20, + "VolumeType": "gp2", }, }, - "ImageId": { - "Ref": "SsmParameterValuetestimageSsmNameC96584B6F00A464EAD1953AFF4B05118Parameter", - }, - "KeyName": { - "Fn::GetAtt": [ - "CiStorageSshIdRsaEC2KeyPairtestcistoragesshidrsa26A4353E", - "KeyPairName", - ], - }, - "MetadataOptions": { - "HttpPutResponseHopLimit": 2, - "HttpTokens": "required", - }, - "SecurityGroupIds": [ - "test-securityGroupId", - ], - "TagSpecifications": [ - { - "ResourceType": "instance", - "Tags": [ - { - "Key": "Name", - "Value": "Test/CiStorage/LaunchTemplate", - }, - ], - }, - { - "ResourceType": "volume", - "Tags": [ - { - "Key": "Name", - "Value": "Test/CiStorage/LaunchTemplate", - }, - ], - }, + ], + "IamInstanceProfile": { + "Ref": "CnstrctStkCnstrctMyCiHost001InstanceInstanceProfile2045E5AE", + }, + "ImageId": { + "Ref": "SsmParameterValuetestimageSsmNameC96584B6F00A464EAD1953AFF4B05118Parameter", + }, + "InstanceType": "t3.large", + "KeyName": { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", ], - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "#cloud-config + }, + "LaunchTemplate": { + "LaunchTemplateName": "StkCnstrctMyCiHost001InstanceLaunchTemplate", + "Version": { + "Fn::GetAtt": [ + "CnstrctStkCnstrctMyCiHost001InstanceLaunchTemplateD3F24170", + "LatestVersionNumber", + ], + }, + }, + "Monitoring": true, + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "CnstrctSgF5C70BA4", + "GroupId", + ], + }, + ], + "SubnetId": { + "Ref": "VpcPrivateSubnet1Subnet536B997A", + }, + "Tags": [ + { + "Key": "Name", + "Value": "my-ci-host-001.test-zoneName", + }, + ], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "", + [ + "#cloud-config timezone: America/Los_Angeles +fqdn: my-ci-host-001.test-zoneName +hostname: my-ci-host-001.test-zoneName +mounts: + - - tmpfs + - /var/lib/docker + - tmpfs + - defaults,noatime,exec,mode=0710,nr_inodes=0,size=4G + - "0" + - "0" apt_sources: - source: deb https://cli.github.com/packages stable main keyid: 23F3D4EA75716059 @@ -782,6 +1195,9 @@ packages: - docker-ce-cli - containerd.io - docker-compose-plugin + - qemu + - qemu-user-static + - binfmt-support - git - gosu - mc @@ -789,35 +1205,78 @@ packages: - apt-transport-https - ca-certificates - tzdata + - atop + - iotop + - htop + - bwm-ng + - jq write_files: - path: /etc/sysctl.d/enable-ipv4-forwarding.conf content: | net.ipv4.conf.all.forwarding=1 - - path: /etc/sysctl.d/lower-fs-inodes-eviction-from-cache.conf + - path: /etc/default/atop + content: | + LOGOPTS="-R" + LOGINTERVAL=15 + LOGGENERATIONS=4 + - path: /etc/environment + append: true content: | - vm.vfs_cache_pressure=0 - vm.swappiness=10 - - path: /var/lib/cloud/scripts/per-once/define-tz-env.sh + TZ="America/Los_Angeles" + - path: /etc/environment + append: true + content: | + LESS="RS" + - path: /etc/docker/daemon.json + permissions: "0644" + content: | + { + "log-driver": "syslog", + "log-opts": { + "tag": "docker/{{.Name}}" + }, + "runtimes": { + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" + } + }, + "default-runtime": "sysbox-runc", + "userns-remap": "sysbox" + } + - path: /var/lib/cloud/scripts/per-once/apply-services-configs.sh permissions: "0755" content: | #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace - - echo 'TZ="America/Los_Angeles"' >> /etc/environment + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + service atop restart || true + sysctl --system - path: /var/lib/cloud/scripts/per-once/increase-docker-shutdown-timeout.sh permissions: "0755" content: | #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace - + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") sed -i -E '/TimeoutStartSec=.*/a TimeoutStopSec=3600' /usr/lib/systemd/system/docker.service systemctl daemon-reload + - path: /var/lib/cloud/scripts/per-once/add-ubuntu-user-to-docker-group-to-access-socket.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -aG docker ubuntu + - path: /var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + systemctl stop docker docker.socket || true + wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" + dpkg -i /tmp/sysbox-ce.deb + rm -f /tmp/sysbox-ce.deb - path: /var/lib/cloud/scripts/per-once/switch-ssm-user-to-ubuntu-on-login.sh permissions: "0755" content: | #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace - + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") echo '[ "$0$@" = "sh" ] && ENV= sudo -u ubuntu -i' > /etc/profile.ssm-user mkdir -p /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/ ( @@ -826,13 +1285,93 @@ write_files: ) > /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/sh-env.conf systemctl daemon-reload systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true - - path: /var/lib/cloud/scripts/per-boot/run-docker-compose-on-boot.sh + - path: /var/lib/cloud/scripts/per-once/rsync-tmpfs-volume-from-old-instance.sh permissions: "0755" content: | #!/bin/bash - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + + instance_id=$(ec2metadata --instance-id) + stack_name=$( + aws ec2 describe-tags \\ + --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:cloudformation:stack-name" \\ + --query "Tags[0].Value" --output text + ) + logical_id=$( + aws ec2 describe-tags \\ + --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:cloudformation:logical-id" \\ + --query "Tags[0].Value" --output text + ) + old_instance_ip_addr=$( + aws ec2 describe-instances \\ + --filters "Name=tag:Name,Values=my-ci-host-001.test-zoneName" "Name=instance-state-name,Values=running" \\ + --query "Reservations[*].Instances[*].[InstanceId,PrivateIpAddress]" --output text \\ + | grep -v "$instance_id" | awk '{print $2}' | head -n1 || true + ) + + if [[ "$old_instance_ip_addr" != "" ]]; then + # Load private key from Secrets Manager to ~/.ssh, to access the old host. + mkdir -p ~/.ssh && chmod 700 ~/.ssh + aws secretsmanager get-secret-value \\ + --secret-id "ec2-ssh-key/", + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private" \\ + --query SecretString --output text \\ + > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + # Stop Docker service on the current host. + systemctl stop docker docker.socket || true + + # Stop Docker service on the old (source) host. + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \\ + "ubuntu@$old_instance_ip_addr" "sudo systemctl stop docker docker.socket || true" + + # 1. Surprisingly, it takes almost the same amount of time to rsync-init + # (if we would run it without stopping Docker on the old host first) + # as to the follow-up rsync-over (after we stopped Docker on the source). + # This is probably because of the RAM drive and large Docker volumes. So + # we skip rsync-init and just go with one full rsync run (with downtime). + # 2. Also, compression (even the fastest one) doesn't speed it up; probably + # because AWS network is faster than instances CPU still. + time rsync \\ + -aHXS --one-file-system --numeric-ids --delete $@ \\ + --rsh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \\ + --rsync-path="sudo rsync" \\ + "ubuntu@$old_instance_ip_addr:/var/lib/docker/" "/var/lib/docker/" + + # We do NOT start Docker service here! Otherwise, it may auto-start some + # containers, those containers will expect the git directory to exist, + # although it may not exist yet. So, we start Docker service in + # run-docker-compose.sh (its 1st run), when we are sure that git is pulled. + fi - echo "*/1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh 2>&1 | logger -t run-docker-compose" > /etc/cron.d/run-docker-compose + aws cloudformation signal-resource \\ + --stack-name "$stack_name" --logical-resource-id "$logical_id" \\ + --unique-id "$instance_id" --status SUCCESS + - path: /etc/rsyslog.d/01-docker-tag-to-serial-console.conf + permissions: "0644" + content: | + if $syslogtag startswith 'docker/' then -/dev/console + # It will also write to /var/log/syslog as usual. + - path: /var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + usermod -a -G tty syslog + systemctl restart rsyslog + - path: /var/lib/cloud/scripts/per-boot/run-docker-compose-on-boot.sh + permissions: "0755" + content: | + #!/bin/bash + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + echo "*/2 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-logs 2>&1 | logger -t docker/run-docker-compose" > /etc/cron.d/run-docker-compose exec /home/ubuntu/run-docker-compose.sh - path: /home/ubuntu/run-docker-compose.sh owner: ubuntu:ubuntu @@ -843,38 +1382,49 @@ write_files: # Switch to non-privileged user if running as root. if [[ $(whoami) != "ubuntu" ]]; then - exec gosu ubuntu:ubuntu "$BASH_SOURCE" + exec gosu ubuntu "$BASH_SOURCE" "$@" fi # Ensure there is only one instance of this script running. exec {FD}<$BASH_SOURCE flock -n "$FD" || { echo "Already running."; exit 0; } - set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace + set -e -o pipefail && echo ================ && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace && export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") + + # Make sure we're using the right timezone; it may be not up + # to date in the current environment during the very 1st run + # from run-docker-compose-on-boot.sh. + source /etc/environment + export TZ # Load private and public keys from Secrets Manager to ~/.ssh. - export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") mkdir -p ~/.ssh && chmod 700 ~/.ssh aws secretsmanager get-secret-value \\ --secret-id "ec2-ssh-key/", - { - "Fn::GetAtt": [ - "CiStorageSshIdRsaEC2KeyPairtestcistoragesshidrsa26A4353E", - "KeyPairName", - ], - }, - "/private" \\ + { + "Fn::GetAtt": [ + "CnstrctSshIdRsaEC2KeyPairstkcnstrctsshidrsaF4F38F69", + "KeyPairName", + ], + }, + "/private" \\ --query SecretString --output text \\ > ~/.ssh/ci-storage chmod 600 ~/.ssh/ci-storage ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub - # Load GitHub PAT from Secrets Manager and login to GitHub. + # Load GitHub PAT from Secrets Manager and log in to GitHub. aws secretsmanager get-secret-value \\ --secret-id "ci-storage/gh-token" \\ --query SecretString --output text \\ | gh auth login --with-token gh auth setup-git + # Log in to ghcr.io every hour. + config=~/.docker/config.json + if [[ ! -f $config ]] || find "$config" -type f -mmin +60 | grep -q .; then + gh auth token | docker login ghcr.io -u "$(gh api user -q .login)" --password-stdin + fi + # Pull the repository. mkdir -p ~/git && cd ~/git if [[ ! -d .git ]]; then @@ -887,199 +1437,85 @@ write_files: git pull --rebase fi - # Run docker compose. - sudo usermod -aG docker ubuntu - export GH_TOKEN + # Process some tokens and print rate limits without xtrace. set +o xtrace GH_TOKEN=$(gh auth token) + echo "Docker Hub Rate Limits:" + docker_hub_token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token || true) + curl -s --head -H "Authorization: Bearer $docker_hub_token" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit || true + echo "GitHub Core Rate Limits:" + gh api -i -X HEAD /rate_limit | grep Ratelimit set -o xtrace - exec sg docker -c 'cd "docker" && docker compose pull && exec docker compose up --build -d' + + # Export env vars for docker compose. + export GH_TOKEN + + # It it's the very 1st run, start Docker service. We do not start it every run, + # because otherwise we wouldn't be able to "systemctl stop docker docker.socket" + # manually or while copying files from the old host. + file=~/.docker-started-after-first-git-clone + if [[ ! -f $file ]]; then + sudo systemctl start docker docker.socket + touch $file + fi + + # Run docker compose. + cd "docker" + docker pull ghcr.io/dimikot/ci-storage:main || true + docker pull ghcr.io/dimikot/ci-runner:main || true + docker compose --profile=ci up --build --remove-orphans -d + sleep 5 + if [[ "$1" != "--no-logs" ]]; then + docker compose logs -n 10 + fi + docker system prune --volumes -f - path: /home/ubuntu/.bash_profile owner: ubuntu:ubuntu permissions: "0644" defer: true content: | #!/bin/bash - if [ -d ~/git/"docker" ]; then + C_CMD="\\033[0;36m" + C_NO="\\033[0m" + if [[ -d ~/git/"docker" ]]; then cd ~/git/"docker" - echo '$ docker compose ps' + echo -e "$C_CMD\\$ docker compose ps$C_NO" docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" - echo + services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) + if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then + cmd="docker compose exec $services bash -l" + echo -e "$C_CMD\\$ $cmd$C_NO" + eval "$cmd" + fi fi ", - ], ], - }, - }, - }, - "LaunchTemplateName": "test-cistorage", - "TagSpecifications": [ - { - "ResourceType": "launch-template", - "Tags": [ - { - "Key": "Name", - "Value": "Test/CiStorage/LaunchTemplate", - }, ], }, - ], + }, }, - "Type": "AWS::EC2::LaunchTemplate", + "Type": "AWS::EC2::Instance", }, - "CiStorageLaunchTemplateProfile1CF26B91": { + "CnstrctStkCnstrctMyCiHost001InstanceInstanceProfile2045E5AE": { "Properties": { "Roles": [ { - "Ref": "CiStorageRunnerRole9B60440E", + "Ref": "CnstrctHostRole5DD9F366", }, ], }, "Type": "AWS::IAM::InstanceProfile", }, - "CiStorageRunnerRole9B60440E": { + "CnstrctStkCnstrctMyCiHost001InstanceLaunchTemplateD3F24170": { "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com", - }, - }, - ], - "Version": "2012-10-17", - }, - "ManagedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":iam::aws:policy/service-role/AmazonEC2RoleforSSM", - ], - ], - }, - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":iam::aws:policy/CloudWatchAgentServerPolicy", - ], - ], - }, - ], - "Policies": [ - { - "PolicyDocument": { - "Statement": [ - { - "Action": "secretsmanager:GetSecretValue", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":secretsmanager:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":secret:ec2-ssh-key/", - { - "Fn::GetAtt": [ - "CiStorageSshIdRsaEC2KeyPairtestcistoragesshidrsa26A4353E", - "KeyPairName", - ], - }, - "/private*", - ], - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CiStorageKeyPairPolicy", - }, - { - "PolicyDocument": { - "Statement": [ - { - "Action": "secretsmanager:GetSecretValue", - "Effect": "Allow", - "Resource": { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition", - }, - ":secretsmanager:", - { - "Ref": "AWS::Region", - }, - ":", - { - "Ref": "AWS::AccountId", - }, - ":secret:ci-storage/gh-token*", - ], - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "PolicyName": "CiStorageGhTokenPolicy", + "LaunchTemplateData": { + "MetadataOptions": { + "HttpTokens": "required", }, - ], - "RoleName": "TestCiStorageRunnerRole", - }, - "Type": "AWS::IAM::Role", - }, - "CiStorageSshIdRsaEC2KeyPairtestcistoragesshidrsa26A4353E": { - "DeletionPolicy": "Delete", - "Properties": { - "Description": "Used to access ci-storage host from self-hosted runner nodes.", - "ExposePublicKey": false, - "KmsPrivate": "alias/aws/secretsmanager", - "KmsPublic": "alias/aws/secretsmanager", - "Name": "test-cistorage-sshidrsa", - "PublicKey": "", - "PublicKeyFormat": "OPENSSH", - "RemoveKeySecretsAfterDays": 0, - "SecretPrefix": "ec2-ssh-key/", - "ServiceToken": { - "Fn::GetAtt": [ - "EC2KeyNameManagerLambdaBE629145", - "Arn", - ], - }, - "StackName": "Test", - "StorePublicKey": false, - "Tags": { - "CreatedByCfnCustomResource": "CFN::Resource::Custom::EC2-Key-Pair", }, + "LaunchTemplateName": "StkCnstrctMyCiHost001InstanceLaunchTemplate", }, - "Type": "Custom::EC2-Key-Pair", - "UpdateReplacePolicy": "Delete", + "Type": "AWS::EC2::LaunchTemplate", }, "EC2KeyNameManagerLambdaBE629145": { "DependsOn": [ @@ -1093,7 +1529,7 @@ write_files: "S3Key": "6df647194cd2bd5032d6a0553b301f3350abb6035c13b5ba2a73503a45e7fd80.zip", }, "Description": "Custom CFN resource: Manage EC2 Key Pairs", - "FunctionName": "Test-CFN-Resource-Custom-EC2-Key-Pair", + "FunctionName": "Stk-CFN-Resource-Custom-EC2-Key-Pair", "Handler": "index.handler", "Role": { "Fn::GetAtt": [ @@ -1258,7 +1694,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc", + "Value": "Stk/Vpc", }, ], }, @@ -1269,7 +1705,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc", + "Value": "Stk/Vpc", }, ], }, @@ -1303,7 +1739,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc/PrivateSubnet1", + "Value": "Stk/Vpc/PrivateSubnet1", }, ], "VpcId": { @@ -1335,7 +1771,7 @@ write_files: }, { "Key": "Name", - "Value": "Test/Vpc/PrivateSubnet1", + "Value": "Stk/Vpc/PrivateSubnet1", }, ], "VpcId": { @@ -1361,7 +1797,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc/PrivateSubnet2", + "Value": "Stk/Vpc/PrivateSubnet2", }, ], "VpcId": { @@ -1404,7 +1840,7 @@ write_files: }, { "Key": "Name", - "Value": "Test/Vpc/PrivateSubnet2", + "Value": "Stk/Vpc/PrivateSubnet2", }, ], "VpcId": { @@ -1434,7 +1870,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc/PublicSubnet1", + "Value": "Stk/Vpc/PublicSubnet1", }, ], }, @@ -1458,7 +1894,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc/PublicSubnet1", + "Value": "Stk/Vpc/PublicSubnet1", }, ], }, @@ -1469,7 +1905,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc/PublicSubnet1", + "Value": "Stk/Vpc/PublicSubnet1", }, ], "VpcId": { @@ -1512,7 +1948,7 @@ write_files: }, { "Key": "Name", - "Value": "Test/Vpc/PublicSubnet1", + "Value": "Stk/Vpc/PublicSubnet1", }, ], "VpcId": { @@ -1542,7 +1978,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc/PublicSubnet2", + "Value": "Stk/Vpc/PublicSubnet2", }, ], }, @@ -1566,7 +2002,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc/PublicSubnet2", + "Value": "Stk/Vpc/PublicSubnet2", }, ], }, @@ -1577,7 +2013,7 @@ write_files: "Tags": [ { "Key": "Name", - "Value": "Test/Vpc/PublicSubnet2", + "Value": "Stk/Vpc/PublicSubnet2", }, ], "VpcId": { @@ -1620,7 +2056,7 @@ write_files: }, { "Key": "Name", - "Value": "Test/Vpc/PublicSubnet2", + "Value": "Stk/Vpc/PublicSubnet2", }, ], "VpcId": { diff --git a/src/internal/__tests__/__snapshots__/buildPercentScalingSteps.test.ts.snap b/src/internal/__tests__/__snapshots__/buildPercentScalingSteps.test.ts.snap new file mode 100644 index 0000000..b389136 --- /dev/null +++ b/src/internal/__tests__/__snapshots__/buildPercentScalingSteps.test.ts.snap @@ -0,0 +1,23 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`buildPercentScalingSteps 1`] = ` +"[ { pctTarget: 70, lower: undefined, upper: 18, change: -74, pctAfter: 69 }, + { pctTarget: 70, lower: 18, upper: 35, change: -50, pctAfter: 70 }, + { pctTarget: 70, lower: 35, upper: 53, change: -24, pctAfter: 70 }, + { pctTarget: 70, lower: 53, upper: 70, change: 0, pctAfter: 70 }, + { pctTarget: 70, lower: 70, upper: 78, change: 11, pctAfter: 70 }, + { pctTarget: 70, lower: 78, upper: 85, change: 21, pctAfter: 70 }, + { pctTarget: 70, lower: 85, upper: 93, change: 33, pctAfter: 70 }, + { pctTarget: 70, lower: 93, upper: undefined, change: 43, pctAfter: 70 } ]" +`; + +exports[`buildPercentScalingSteps 2`] = ` +"[ { pctTarget: 50, lower: undefined, upper: 13, change: -74, pctAfter: 50 }, + { pctTarget: 50, lower: 13, upper: 25, change: -50, pctAfter: 50 }, + { pctTarget: 50, lower: 25, upper: 38, change: -24, pctAfter: 50 }, + { pctTarget: 50, lower: 38, upper: 50, change: 0, pctAfter: 50 }, + { pctTarget: 50, lower: 50, upper: 63, change: 26, pctAfter: 50 }, + { pctTarget: 50, lower: 63, upper: 75, change: 50, pctAfter: 50 }, + { pctTarget: 50, lower: 75, upper: 88, change: 76, pctAfter: 50 }, + { pctTarget: 50, lower: 88, upper: undefined, change: 100, pctAfter: 50 } ]" +`; diff --git a/src/internal/__tests__/buildPercentScalingSteps.test.ts b/src/internal/__tests__/buildPercentScalingSteps.test.ts new file mode 100644 index 0000000..ae09115 --- /dev/null +++ b/src/internal/__tests__/buildPercentScalingSteps.test.ts @@ -0,0 +1,11 @@ +import { inspect } from "util"; +import { buildPercentScalingSteps } from "../buildPercentScalingSteps"; + +test("buildPercentScalingSteps", () => { + expect( + inspect(buildPercentScalingSteps(70, 4), { compact: true }), + ).toMatchSnapshot(); + expect( + inspect(buildPercentScalingSteps(50, 4), { compact: true }), + ).toMatchSnapshot(); +}); diff --git a/src/internal/__tests__/namer.test.ts b/src/internal/__tests__/namer.test.ts new file mode 100644 index 0000000..ace483d --- /dev/null +++ b/src/internal/__tests__/namer.test.ts @@ -0,0 +1,7 @@ +import { namer } from "../namer"; + +test("namer", () => { + expect(namer("OneTwo" as any).kebab).toBe("one-two"); + expect(namer("one-two" as any).pascal).toBe("OneTwo"); + expect(namer("one", "two").pascal).toBe("OneTwo"); +}); diff --git a/src/internal/buildPercentScalingSteps.ts b/src/internal/buildPercentScalingSteps.ts new file mode 100644 index 0000000..3abdd88 --- /dev/null +++ b/src/internal/buildPercentScalingSteps.ts @@ -0,0 +1,62 @@ +type PercentScalingStep = { + pctTarget: number; + upper?: number; + lower?: number; + change: number; + pctAfter: number; +}; + +/** + * Builds scaling steps, so that the further the actual percentage metric is + * from pctTarget, the more instances will be added or removed (so the resulting + * instances busy percentage will remain around pctTarget if nothing changes). + */ +export function buildPercentScalingSteps( + pctTarget: number, + steps: number, +): PercentScalingStep[] { + const result: PercentScalingStep[] = []; + + // An example number of instances to illustrate the "after" situation. + const N = 100; + + { + const stepSize = pctTarget / steps; + for (let i = steps; i >= 1; i--) { + const lower = Math.round(pctTarget - i * stepSize); + const upper = Math.round(pctTarget - (i - 1) * stepSize); + const change = Math.round((upper / pctTarget - 1) * 100); + const pctAfter = Math.round( + ((N * upper * 0.01) / (N + N * change * 0.01)) * 100, + ); + result.push({ + pctTarget, + lower: lower <= 0 ? undefined : lower, + upper, + change, + pctAfter, + }); + } + } + + { + const stepSize = (100 - pctTarget) / steps; + for (let i = 1; i <= steps; i++) { + const lower = Math.round(pctTarget + (i - 1) * stepSize); + const upper = Math.round(pctTarget + i * stepSize); + const change = Math.round((upper / pctTarget - 1) * 100); + const pctAfter = Math.round( + ((N * upper * 0.01) / (N + N * change * 0.01)) * 100, + ); + result.push({ + pctTarget, + lower, + upper: upper >= 100 ? undefined : upper, + change, + pctAfter, + }); + } + } + + return result; +} diff --git a/src/internal/cloudConfigBuild.ts b/src/internal/cloudConfigBuild.ts index a0012a5..e96e437 100644 --- a/src/internal/cloudConfigBuild.ts +++ b/src/internal/cloudConfigBuild.ts @@ -3,22 +3,30 @@ import { dedent } from "./dedent"; /** * Builds a reusable and never changing cloud config to be passed to the - * instance's CloudInit service. + * instance's CloudInit service. This config is multi-purpose: it doesn't know + * about the role of the instance (host or runner), it just initiates the + * instance to run docker-compose file on it. */ export function cloudConfigBuild({ fqdn, ghTokenSecretName, + dockerComposeEnv, + dockerComposeProfiles, ghDockerComposeDirectoryUrl, keyPairPrivateKeySecretName, timeZone, - mount, + tmpfs, + swapSizeGb, }: { - fqdn: string; + fqdn: string | undefined; ghTokenSecretName: string; ghDockerComposeDirectoryUrl: string; + dockerComposeEnv: Record; + dockerComposeProfiles: string[]; keyPairPrivateKeySecretName: string; timeZone: string | undefined; - mount: { volumeId: string; path: string } | undefined; + tmpfs: { path: string; maxSizeGb?: number } | undefined; + swapSizeGb: number | undefined; }) { if (!ghDockerComposeDirectoryUrl.match(/^([^#]+)(?:#([^:]*):(.*))?$/s)) { throw ( @@ -30,12 +38,38 @@ export function cloudConfigBuild({ const repoUrl = RegExp.$1; const branch = RegExp.$2 || ""; const path = (RegExp.$3 || ".").replace(/^\/+|\/+$/gs, ""); - const preamble = - 'set -e -o pipefail && echo --- && echo "Running $BASH_SOURCE as $(whoami)" && set -o xtrace'; + const preamble = [ + "set -e -o pipefail", + "echo ================", + 'echo "Running $BASH_SOURCE as $(whoami)"', + "set -o xtrace", + 'export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//")', + ].join(" && "); return { timezone: timeZone, fqdn: fqdn || undefined, + hostname: fqdn || undefined, + swap: swapSizeGb + ? { + filename: "/var/swapfile", + size: "auto", + maxsize: 1024 * 1024 * 1024 * swapSizeGb, + } + : undefined, + mounts: tmpfs + ? [ + [ + "tmpfs", + tmpfs.path, + "tmpfs", + "defaults,noatime,exec,mode=0710,nr_inodes=0" + + (tmpfs.maxSizeGb ? `,size=${tmpfs.maxSizeGb}G` : ""), + "0", + "0", + ], + ] + : undefined, apt_sources: [ { source: "deb https://cli.github.com/packages stable main", @@ -55,6 +89,9 @@ export function cloudConfigBuild({ "docker-ce-cli", "containerd.io", "docker-compose-plugin", + "qemu", + "qemu-user-static", + "binfmt-support", "git", "gosu", "mc", @@ -62,6 +99,11 @@ export function cloudConfigBuild({ "apt-transport-https", "ca-certificates", "tzdata", + "atop", + "iotop", + "htop", + "bwm-ng", + "jq", ], write_files: compact([ { @@ -71,20 +113,54 @@ export function cloudConfigBuild({ `), }, { - path: "/etc/sysctl.d/lower-fs-inodes-eviction-from-cache.conf", + path: "/etc/default/atop", content: dedent(` - vm.vfs_cache_pressure=0 - vm.swappiness=10 + LOGOPTS="-R" + LOGINTERVAL=15 + LOGGENERATIONS=4 `), }, timeZone && { - path: "/var/lib/cloud/scripts/per-once/define-tz-env.sh", + path: "/etc/environment", + append: true, + content: dedent(` + TZ="${timeZone}" + `), + }, + { + path: "/etc/environment", + append: true, + content: dedent(` + LESS="RS" + `), + }, + { + path: "/etc/docker/daemon.json", + permissions: "0644", + content: dedent(` + { + "log-driver": "syslog", + "log-opts": { + "tag": "docker/{{.Name}}" + }, + "runtimes": { + "sysbox-runc": { + "path": "/usr/bin/sysbox-runc" + } + }, + "default-runtime": "sysbox-runc", + "userns-remap": "sysbox" + } + `), + }, + { + path: "/var/lib/cloud/scripts/per-once/apply-services-configs.sh", permissions: "0755", content: dedent(` #!/bin/bash ${preamble} - - echo 'TZ="${timeZone}"' >> /etc/environment + service atop restart || true + sysctl --system `), }, { @@ -93,18 +169,37 @@ export function cloudConfigBuild({ content: dedent(` #!/bin/bash ${preamble} - sed -i -E '/TimeoutStartSec=.*/a TimeoutStopSec=3600' /usr/lib/systemd/system/docker.service systemctl daemon-reload `), }, + { + path: "/var/lib/cloud/scripts/per-once/add-ubuntu-user-to-docker-group-to-access-socket.sh", + permissions: "0755", + content: dedent(` + #!/bin/bash + ${preamble} + usermod -aG docker ubuntu + `), + }, + { + path: "/var/lib/cloud/scripts/per-once/install-sysbox-for-docker-in-docker.sh", + permissions: "0755", + content: dedent(` + #!/bin/bash + ${preamble} + systemctl stop docker docker.socket || true + wget -nv -O /tmp/sysbox-ce.deb "https://downloads.nestybox.com/sysbox/releases/v0.6.4/sysbox-ce_0.6.4-0.linux_$(dpkg --print-architecture).deb" + dpkg -i /tmp/sysbox-ce.deb + rm -f /tmp/sysbox-ce.deb + `), + }, { path: "/var/lib/cloud/scripts/per-once/switch-ssm-user-to-ubuntu-on-login.sh", permissions: "0755", content: dedent(` #!/bin/bash ${preamble} - echo '[ "$0$@" = "sh" ] && ENV= sudo -u ubuntu -i' > /etc/profile.ssm-user mkdir -p /etc/systemd/system/snap.amazon-ssm-agent.amazon-ssm-agent.service.d/ ( @@ -115,89 +210,88 @@ export function cloudConfigBuild({ systemctl restart snap.amazon-ssm-agent.amazon-ssm-agent.service || true `), }, - mount && { - path: "/var/lib/cloud/scripts/per-once/detach-volume-from-old-instance-and-mount.sh", - permissions: "0755", - content: dedent(` - #!/bin/bash - ${preamble} + tmpfs && + fqdn && { + path: "/var/lib/cloud/scripts/per-once/rsync-tmpfs-volume-from-old-instance.sh", + permissions: "0755", + content: dedent(` + #!/bin/bash + ${preamble} - export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") - volume_id="${mount.volumeId}" - volume_hash="\${volume_id##vol-}" - volume_dir="/mnt" - volume_label="MNT" - instance_id=$(ec2metadata --instance-id) + instance_id=$(ec2metadata --instance-id) + stack_name=$( + aws ec2 describe-tags \\ + --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:cloudformation:stack-name" \\ + --query "Tags[0].Value" --output text + ) + logical_id=$( + aws ec2 describe-tags \\ + --filters "Name=resource-id,Values=$instance_id" "Name=key,Values=aws:cloudformation:logical-id" \\ + --query "Tags[0].Value" --output text + ) + old_instance_ip_addr=$( + aws ec2 describe-instances \\ + --filters "Name=tag:Name,Values=${fqdn}" "Name=instance-state-name,Values=running" \\ + --query "Reservations[*].Instances[*].[InstanceId,PrivateIpAddress]" --output text \\ + | grep -v "$instance_id" | awk '{print $2}' | head -n1 || true + ) - # Stop the old instances. This causes a small downtime of the host - # service, but it's acceptable for the CI use case. - old_instance_id=$( - aws ec2 describe-volumes \\ - --volume-ids "$volume_id" \\ - --query "Volumes[].Attachments[].InstanceId" \\ - --output text - ) - if [[ "$old_instance_id" != "" ]]; then - sent_command=0 - while ! aws ec2 describe-instances \\ - --instance-ids "$old_instance_id" \\ - --query "Reservations[].Instances[].State.Name" \\ - --output text \\ - | egrep -q "stopped|terminated" - do - if [[ "$sent_command" == "0" ]]; then - sent_command=1 - aws ec2 stop-instances --instance-ids "$old_instance_id" || true - fi - sleep 1 - done - fi + if [[ "$old_instance_ip_addr" != "" ]]; then + # Load private key from Secrets Manager to ~/.ssh, to access the old host. + mkdir -p ~/.ssh && chmod 700 ~/.ssh + aws secretsmanager get-secret-value \\ + --secret-id "${keyPairPrivateKeySecretName}" \\ + --query SecretString --output text \\ + > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa - # Detach volume from the old instance. - sent_command=0 - while ! aws ec2 describe-volumes \\ - --volume-ids "$volume_id" \\ - --query "Volumes[].State" \\ - --output text \\ - | grep -q available - do - if [[ "$sent_command" == "0" ]]; then - sent_command=1 - aws ec2 detach-volume --volume-id "$volume_id" --force || true - fi - sleep 0.2; - done + # Stop Docker service on the current host. + systemctl stop docker docker.socket || true - # Attach volume to this instance and wait for the device to appear. - sent_command=0 - while ! ls /dev/disk/by-id | grep -q "$volume_hash"; do - if [[ "$sent_command" == "0" ]]; then - sent_command=1 - aws ec2 attach-volume --volume-id "$volume_id" --instance-id "$instance_id" --device /dev/sdf + # Stop Docker service on the old (source) host. + ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \\ + "ubuntu@$old_instance_ip_addr" "sudo systemctl stop docker docker.socket || true" + + # 1. Surprisingly, it takes almost the same amount of time to rsync-init + # (if we would run it without stopping Docker on the old host first) + # as to the follow-up rsync-over (after we stopped Docker on the source). + # This is probably because of the RAM drive and large Docker volumes. So + # we skip rsync-init and just go with one full rsync run (with downtime). + # 2. Also, compression (even the fastest one) doesn't speed it up; probably + # because AWS network is faster than instances CPU still. + time rsync \\ + -aHXS --one-file-system --numeric-ids --delete $@ \\ + --rsh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \\ + --rsync-path="sudo rsync" \\ + "ubuntu@$old_instance_ip_addr:${tmpfs.path}/" "${tmpfs.path}/" + + # We do NOT start Docker service here! Otherwise, it may auto-start some + # containers, those containers will expect the git directory to exist, + # although it may not exist yet. So, we start Docker service in + # run-docker-compose.sh (its 1st run), when we are sure that git is pulled. fi - sleep 0.2 - done - # Mount volume if it already exists, or create the filesystem. - lsblk - ls -la /dev/disk/by-id - device=$(echo /dev/disk/by-id/*$volume_hash) - if ! grep -q "LABEL=$volume_label" /etc/fstab; then - echo "LABEL=$volume_label $volume_dir auto defaults,noatime,data=writeback 0 0" >> /etc/fstab - fi - mount -a || true - if ! mountpoint "$volume_dir"; then - mkfs -t ext4 "$device" - tune2fs -L "$volume_label" "$device" - mount -a - systemctl stop docker docker.socket - ls -la /var/lib/docker - cp -axT /var/lib/docker "$volume_dir/var_lib_docker" - mv -f /var/lib/docker /var/lib/docker.old - ln -sT "$volume_dir/var_lib_docker" /var/lib/docker - systemctl start docker docker.socket - fi - ls -la "$volume_dir" + aws cloudformation signal-resource \\ + --stack-name "$stack_name" --logical-resource-id "$logical_id" \\ + --unique-id "$instance_id" --status SUCCESS + `), + }, + { + path: "/etc/rsyslog.d/01-docker-tag-to-serial-console.conf", + permissions: "0644", + content: dedent(` + if $syslogtag startswith 'docker/' then -/dev/console + # It will also write to /var/log/syslog as usual. + `), + }, + { + path: "/var/lib/cloud/scripts/per-once/allow-rsyslog-write-to-serial-console.sh", + permissions: "0755", + content: dedent(` + #!/bin/bash + ${preamble} + usermod -a -G tty syslog + systemctl restart rsyslog `), }, { @@ -206,8 +300,7 @@ export function cloudConfigBuild({ content: dedent(` #!/bin/bash ${preamble} - - echo "*/1 * * * * ubuntu /home/ubuntu/run-docker-compose.sh 2>&1 | logger -t run-docker-compose" > /etc/cron.d/run-docker-compose + echo "*/2 * * * * ubuntu /home/ubuntu/run-docker-compose.sh --no-logs 2>&1 | logger -t docker/run-docker-compose" > /etc/cron.d/run-docker-compose exec /home/ubuntu/run-docker-compose.sh `), }, @@ -221,7 +314,7 @@ export function cloudConfigBuild({ # Switch to non-privileged user if running as root. if [[ $(whoami) != "ubuntu" ]]; then - exec gosu ubuntu:ubuntu "$BASH_SOURCE" + exec gosu ubuntu "$BASH_SOURCE" "$@" fi # Ensure there is only one instance of this script running. @@ -229,8 +322,13 @@ export function cloudConfigBuild({ flock -n "$FD" || { echo "Already running."; exit 0; } ${preamble} + # Make sure we're using the right timezone; it may be not up + # to date in the current environment during the very 1st run + # from run-docker-compose-on-boot.sh. + source /etc/environment + export TZ + # Load private and public keys from Secrets Manager to ~/.ssh. - export AWS_DEFAULT_REGION=$(ec2metadata --availability-zone | sed "s/[a-z]$//") mkdir -p ~/.ssh && chmod 700 ~/.ssh aws secretsmanager get-secret-value \\ --secret-id "${keyPairPrivateKeySecretName}" \\ @@ -239,13 +337,19 @@ export function cloudConfigBuild({ chmod 600 ~/.ssh/ci-storage ssh-keygen -f ~/.ssh/ci-storage -y > ~/.ssh/ci-storage.pub - # Load GitHub PAT from Secrets Manager and login to GitHub. + # Load GitHub PAT from Secrets Manager and log in to GitHub. aws secretsmanager get-secret-value \\ --secret-id "${ghTokenSecretName}" \\ --query SecretString --output text \\ | gh auth login --with-token gh auth setup-git + # Log in to ghcr.io every hour. + config=~/.docker/config.json + if [[ ! -f $config ]] || find "$config" -type f -mmin +60 | grep -q .; then + gh auth token | docker login ghcr.io -u "$(gh api user -q .login)" --password-stdin + fi + # Pull the repository. mkdir -p ~/git && cd ~/git if [[ ! -d .git ]]; then @@ -258,13 +362,41 @@ export function cloudConfigBuild({ git pull --rebase fi - # Run docker compose. - sudo usermod -aG docker ubuntu - export GH_TOKEN + # Process some tokens and print rate limits without xtrace. set +o xtrace GH_TOKEN=$(gh auth token) + echo "Docker Hub Rate Limits:" + docker_hub_token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token || true) + curl -s --head -H "Authorization: Bearer $docker_hub_token" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit || true + echo "GitHub Core Rate Limits:" + gh api -i -X HEAD /rate_limit | grep Ratelimit set -o xtrace - exec sg docker -c 'cd "${path}" && docker compose pull && exec docker compose up --build -d' + + # Export env vars for docker compose. + export GH_TOKEN + ${Object.entries(dockerComposeEnv) + .map(([k, v]) => `export ${k}="${v}"`) + .join("\n")} + + # It it's the very 1st run, start Docker service. We do not start it every run, + # because otherwise we wouldn't be able to "systemctl stop docker docker.socket" + # manually or while copying files from the old host. + file=~/.docker-started-after-first-git-clone + if [[ ! -f $file ]]; then + sudo systemctl start docker docker.socket + touch $file + fi + + # Run docker compose. + cd "${path}" + docker pull ghcr.io/dimikot/ci-storage:main || true + docker pull ghcr.io/dimikot/ci-runner:main || true + docker compose ${dockerComposeProfiles.map((profile) => `--profile=${profile} `).join("")}up --build --remove-orphans -d + sleep 5 + if [[ "$1" != "--no-logs" ]]; then + docker compose logs -n 10 + fi + docker system prune --volumes -f `), }, { @@ -274,11 +406,18 @@ export function cloudConfigBuild({ defer: true, content: dedent(` #!/bin/bash - if [ -d ~/git/"${path}" ]; then + C_CMD="\\033[0;36m" + C_NO="\\033[0m" + if [[ -d ~/git/"${path}" ]]; then cd ~/git/"${path}" - echo '$ docker compose ps' + echo -e "$C_CMD\\$ docker compose ps$C_NO" docker --log-level=ERROR compose ps --format="table {{.Service}}\\t{{.Status}}\\t{{.Ports}}" - echo + services=$(docker compose ps --format '{{.Service}}' 2>/dev/null) + if [[ "$services" != "" && $(echo "$services" | wc -l) -eq 1 ]]; then + cmd="docker compose exec $services bash -l" + echo -e "$C_CMD\\$ $cmd$C_NO" + eval "$cmd" + fi fi `), }, diff --git a/src/internal/dedent.ts b/src/internal/dedent.ts index 06d8496..405cee7 100644 --- a/src/internal/dedent.ts +++ b/src/internal/dedent.ts @@ -1,11 +1,16 @@ /** - * Removes leading indentation from all lines of the text. + * Removes leading indentation from each line of the text. Also, it some line + * contains only the indentation spaces immediately followed by \n, the line is + * removed entirely. */ export function dedent(text: string): string { text = text.replace(/^([ \t\r]*\n)+/s, "").trimEnd(); - const matches = text.match(/^[ \t]+/s); + const spacePrefix = text.match(/^([ \t]+)/s) ? RegExp.$1 : null; return ( - (matches ? text.replace(new RegExp("^" + matches[0], "mg"), "") : text) + - "\n" + (spacePrefix + ? text + .replace(new RegExp(`^${spacePrefix}\n`, "mg"), "") + .replace(new RegExp(`^${spacePrefix}`, "mg"), "") + : text) + "\n" ); }