From d7ea90e80eb6eeeb3ae7a519e11990e4f5d6b086 Mon Sep 17 00:00:00 2001 From: Tim Stibbs Date: Wed, 15 Jan 2025 19:43:31 +0000 Subject: [PATCH] Migrating code to aws sdk v3 --- TODO.md | 16 +++++ backend/package-lock.json | 142 ++++++++++++++++++++++++++++++++++++++ backend/package.json | 1 + backend/tools/upload.js | 29 ++++---- 4 files changed, 174 insertions(+), 14 deletions(-) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..f8f61d8 --- /dev/null +++ b/TODO.md @@ -0,0 +1,16 @@ +These three entries are duplicated because of an old and new lighthouse - however the old lighthouse doesn't appear to still exist. They should have a demolished date on them, and then we should filter out entries with that. There might be more than just these three that get filtered out. + +Fastnet Lighthouse +Hale Head Lighthouse +Hoxa Head Lighthouse + +https://www.wikidata.org/wiki/Q26161442 +"dissolved, abolished or demolished date" + +also demolished: +Seaton Carew lighthouse + +TODO: + +- order the things in the outputted datasource, so that things with duplicate names can't keep swapping places and causing spurious entries in the report +- include the wikidata id in the outputted data, it's no use to the front-end but is useful to the data process, so worth the slight increase in data diff --git a/backend/package-lock.json b/backend/package-lock.json index 96bf5ac..dc0c88f 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -12,6 +12,7 @@ "devDependencies": { "@aws-sdk/client-cloudformation": "^3.726.1", "@aws-sdk/client-s3": "^3.726.1", + "@aws-sdk/lib-storage": "^3.726.1", "@tstibbs/cloud-core-utils": "file:../../cloud-core/aws/utils", "aws-cdk": "2.140.0", "aws-cdk-lib": "2.140.0", @@ -29,6 +30,8 @@ "dev": true, "dependencies": { "@aws-sdk/client-s3": "^3.577.0", + "@aws-sdk/lib-storage": "^3.726.1", + "@aws-sdk/s3-request-presigner": "^3.726.1", "aws-cdk": "^2.140.0", "aws-cdk-lib": "^2.140.0", "dotenv": "^16.4.5", @@ -733,6 +736,27 @@ "@aws-sdk/client-sts": "^3.723.0" } }, + "node_modules/@aws-sdk/lib-storage": { + "version": "3.726.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.726.1.tgz", + "integrity": "sha512-WuDxSZ8Bfe1N7gn5eXQ02dhlKWCAwW5qQErpJ4CCddXosF+gLxhGkrP9LkaaP0CpA3PxboHyET6HbWAggOWtqA==", + "dev": true, + "dependencies": { + "@smithy/abort-controller": "^4.0.0", + "@smithy/middleware-endpoint": "^4.0.0", + "@smithy/smithy-client": "^4.0.0", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.726.1" + } + }, "node_modules/@aws-sdk/middleware-bucket-endpoint": { "version": "3.726.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.726.0.tgz", @@ -3869,6 +3893,26 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", @@ -3938,6 +3982,16 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "dev": true, + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -4320,6 +4374,15 @@ "node": ">=4" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -4595,6 +4658,26 @@ "node": ">=10.17.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/import-local": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", @@ -5871,6 +5954,20 @@ "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==", "dev": true }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -5927,6 +6024,26 @@ "node": ">=10" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -6015,6 +6132,25 @@ "node": ">=10" } }, + "node_modules/stream-browserify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", + "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", + "dev": true, + "dependencies": { + "inherits": "~2.0.4", + "readable-stream": "^3.5.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-length": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", @@ -6218,6 +6354,12 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, "node_modules/v8-to-istanbul": { "version": "9.1.3", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.3.tgz", diff --git a/backend/package.json b/backend/package.json index 2c388e8..de45601 100644 --- a/backend/package.json +++ b/backend/package.json @@ -9,6 +9,7 @@ "devDependencies": { "@aws-sdk/client-cloudformation": "^3.726.1", "@aws-sdk/client-s3": "^3.726.1", + "@aws-sdk/lib-storage": "^3.726.1", "@tstibbs/cloud-core-utils": "file:../../cloud-core/aws/utils", "aws-cdk": "2.140.0", "aws-cdk-lib": "2.140.0", diff --git a/backend/tools/upload.js b/backend/tools/upload.js index 69fde51..21a1e58 100644 --- a/backend/tools/upload.js +++ b/backend/tools/upload.js @@ -1,19 +1,17 @@ import {strict as assert} from 'assert' import {createReadStream} from 'fs' -import aws from 'aws-sdk' +import {CloudFormation} from '@aws-sdk/client-cloudformation' +import {Upload} from '@aws-sdk/lib-storage' +import {S3} from '@aws-sdk/client-s3' +import {defaultAwsClientConfig} from '@tstibbs/cloud-core-utils/src/tools/aws-client-config.js' import {STACK_NAME} from '../lib/deploy-envs.js' const GB_DATA_PATH = 'data' -aws.config.region = 'eu-west-2' -aws.config.apiVersions = { - s3: '2006-03-01', - cloudformation: '2010-05-15' -} -let cloudformation = new aws.CloudFormation() -let s3 = new aws.S3() +let cloudformation = new CloudFormation(defaultAwsClientConfig) +let s3 = new S3(defaultAwsClientConfig) const args = process.argv.slice(2) if (args.length < 1) { @@ -44,12 +42,15 @@ async function upload(prefix, fileName, body, contentType) { key = fileName } console.log(`Uploading ${key}`) - let uploadResponse = await s3.upload({ - Bucket: bucketName, - Key: key, - Body: body, - ContentType: contentType - }) + let uploadResponse = await new Upload({ + client: s3, + params: { + Bucket: bucketName, + Key: key, + Body: body, + ContentType: contentType + } + }).done() assert.notEqual(uploadResponse.Location, null) assert.notEqual(uploadResponse.Location, undefined)