From 7be188eff599f51e813e0514f097a6681a182111 Mon Sep 17 00:00:00 2001 From: Shubham Kanodia Date: Tue, 17 Sep 2019 23:19:12 +0530 Subject: [PATCH] Progressive loading of exports sizes, as well as dedupe jobs by both type and id --- .../BuildProgressIndicator.js | 2 +- .../ProgressHex/progress-hex-timeline.js | 3 +- .../ExportAnalysisSection.js | 32 +++++++++++-------- process.yml | 2 +- server/Queue.js | 30 ++++++++--------- server/config.js | 3 +- server/middlewares/rateLimit.middleware.js | 2 +- .../middlewares/results/error.middleware.js | 4 ++- 8 files changed, 44 insertions(+), 34 deletions(-) diff --git a/client/components/BuildProgressIndicator/BuildProgressIndicator.js b/client/components/BuildProgressIndicator/BuildProgressIndicator.js index f953ac34..6c725d35 100644 --- a/client/components/BuildProgressIndicator/BuildProgressIndicator.js +++ b/client/components/BuildProgressIndicator/BuildProgressIndicator.js @@ -1,7 +1,7 @@ import React, { Component } from 'react' -import './BuildProgressIndicator.scss' import ProgressHex from '../ProgressHex' +import './BuildProgressIndicator.scss' export default class BuildProgressIndicator extends Component { constructor(props) { diff --git a/client/components/ProgressHex/progress-hex-timeline.js b/client/components/ProgressHex/progress-hex-timeline.js index 3c78e9b4..b942c5d1 100644 --- a/client/components/ProgressHex/progress-hex-timeline.js +++ b/client/components/ProgressHex/progress-hex-timeline.js @@ -102,6 +102,7 @@ class Trailblaze { createTrail() { const line = document.createElementNS('http://www.w3.org/2000/svg', 'line') + line.setAttribute('stroke-width', '0.5') line.setAttribute('class', 'progress-hex__trail') return line } @@ -168,7 +169,7 @@ class Trailblaze { anime({ targets: this.lines, opacity: [1, 0.9, 0], - strokeDashoffset: [(el) => anime.setDashoffset(el), 0], + strokeDashoffset: [(el) => el && anime.setDashoffset(el), 0], x1: el => lineMap.get(el).source.cx, x2: el => lineMap.get(el).destination.cx, y1: el => lineMap.get(el).source.cy, diff --git a/pages/result/components/ExportAnalysisSection/ExportAnalysisSection.js b/pages/result/components/ExportAnalysisSection/ExportAnalysisSection.js index 8ff99fe1..3beeed31 100644 --- a/pages/result/components/ExportAnalysisSection/ExportAnalysisSection.js +++ b/pages/result/components/ExportAnalysisSection/ExportAnalysisSection.js @@ -11,7 +11,8 @@ import './ExportAnalysisSection.scss' const State = { TBD: 'tbd', IN_PROGRESS: 'in-progress', - FULFILLED: 'fulfilled', + EXPORTS_FULFILLED: 'exports-fulfilled', + SIZES_FULFILLED: 'sizes-fulfilled', REJECTED: 'rejected' } @@ -67,6 +68,7 @@ class ExportPill extends React.Component { function ExportList({ exports, totalSize, isLoading }) { const shouldShowLabels = exports.length > 20 const exportDictionary = {} + let curIndex = 0 exports.forEach(exp => { const firstLetter = exp.name[0].toLowerCase() @@ -82,7 +84,7 @@ function ExportList({ exports, totalSize, isLoading }) { { Object.keys(exportDictionary) .sort() - .map((letter, letterIndex) => ( + .map(letter => (
{shouldShowLabels && ( @@ -94,6 +96,7 @@ function ExportList({ exports, totalSize, isLoading }) { name={exportDictionary[letter][0].name} path={exportDictionary[letter][0].path} key={exportDictionary[letter][0].name} + isLoading={curIndex++ < 40 && isLoading} />
)} @@ -104,7 +107,7 @@ function ExportList({ exports, totalSize, isLoading }) { name={exp.name} path={exp.path} key={exp.name} - isLoading={(letterIndex * expIndex) < 20 && isLoading} + isLoading={curIndex++ < 40 && isLoading} /> ))} @@ -155,7 +158,10 @@ class ExportAnalysisSection extends Component { API.getExports(packageString) .then((results) => { - this.setState({ exports: results.exports }) + this.setState({ + exports: results.exports, + analysisState: State.EXPORTS_FULFILLED, + }) Analytics.event({ category: 'Export Analysis', @@ -173,7 +179,7 @@ class ExportAnalysisSection extends Component { .then(() => API.getExportsSizes(packageString)) .then((results) => { this.setState({ - analysisState: State.FULFILLED, + analysisState: State.SIZES_FULFILLED, assets: results.assets.map(asset => ({ ...asset, path: this.state.exports[asset.name] })) }) @@ -233,9 +239,9 @@ class ExportAnalysisSection extends Component { renderSuccess() { const { result } = this.props const { gzip: totalSize } = result - const { exports, analysisState, assets, filterText, resultError } = this.state + const { exports, analysisState, assets, filterText } = this.state - const normalizedExports = analysisState === State.FULFILLED ? assets : + const normalizedExports = analysisState === State.SIZES_FULFILLED ? assets : Object.keys(exports) .filter(exp => !exp.startsWith('_')) .map((exp) => ({ name: exp })) @@ -261,7 +267,7 @@ class ExportAnalysisSection extends Component { @@ -290,12 +296,12 @@ class ExportAnalysisSection extends Component { return (

Exports Analysis

+ + {this.getIncompatibleMessage() && this.renderIncompatible()} + {analysisState === State.REJECTED && this.renderFailure()} { - this.getIncompatibleMessage() ? this.renderIncompatible() : - (analysisState === State.IN_PROGRESS ? this.renderProgress() : - analysisState === State.REJECTED ? this.renderFailure() : - analysisState === State.FULFILLED ? this.renderSuccess() : null - ) + (analysisState === State.EXPORTS_FULFILLED || analysisState === State.SIZES_FULFILLED) ? + this.renderSuccess() : this.renderProgress() }
); diff --git a/process.yml b/process.yml index 85f5274d..ed278461 100644 --- a/process.yml +++ b/process.yml @@ -12,7 +12,7 @@ apps: - script: build-service/index.js name: build-service instances: 3 - max_memory_restart: 800M + max_memory_restart: 1000M error_file: logs/build-service.log out_file: logs/build-service.log env: diff --git a/server/Queue.js b/server/Queue.js index fd125252..aeb3357b 100644 --- a/server/Queue.js +++ b/server/Queue.js @@ -41,8 +41,8 @@ class Queue { this.executorMap[jobType] = handler } - hasJob(id) { - return this.jobs.some(job => job.id === id) + hasJob(id, type) { + return this.jobs.some(job => job.id === id && type === job.type) } getRunningJobs() { @@ -113,9 +113,9 @@ class Queue { ) } - removeJob(id) { + removeJob(id, type) { this.jobs = - this.jobs.filter(job => job.id !== id) + this.jobs.filter(job => (job.id !== id && job.type !== type)) } /** @@ -138,9 +138,9 @@ class Queue { this.jobs = [] } - setJobToProcessing(id) { + setJobToProcessing(id, type) { this.jobs.forEach((job) => { - if (job.id === id) { + if (job.id === id && job.type === type) { job.status = Job.status.PROCESSING } }) @@ -172,33 +172,33 @@ class Queue { const nextJob = this.getNextJobToRun() log('executing job ... %o', { id: nextJob.id, type: nextJob.type, priority: nextJob.priority }) - this.setJobToProcessing(nextJob.id) + this.setJobToProcessing(nextJob.id, nextJob.type) const callResult = this.executorMap[nextJob.type].call(this, nextJob.params) Promise.resolve(callResult) .then((result) => { - log('job %s was a success, removing it', nextJob.id) + log('job %s was a success, removing it', nextJob.id, nextJob.type) nextJob.successListeners.forEach((listener) => { listener.call(this, result) }) - this.removeJob(nextJob.id) + this.removeJob(nextJob.id, nextJob.type) this.executeNextJobIfPossible() }) .catch((err) => { - log('job %s was a failure, removing it', nextJob.id) + log('job %s was a failure, removing it', nextJob.id, nextJob.type) nextJob.failureListeners.forEach((listener) => { listener.call(this, err) }) - this.removeJob(nextJob.id) + this.removeJob(nextJob.id, nextJob.type) this.executeNextJobIfPossible() }) } - addListenersToJob(id, { resolve, reject }) { + addListenersToJob(id, type, { resolve, reject }) { this.jobs.forEach((job) => { - if (job.id === id) { + if (job.id === id && job.type === type) { job.successListeners.push(resolve); job.failureListeners.push(reject); } @@ -228,9 +228,9 @@ class Queue { // If a job with the given id already exists, // just add the success / failure callbacks to // that existing job (dedup) - if (this.hasJob(id)) { + if (this.hasJob(id, type)) { log('job id %s already present, adding callbacks', id) - this.addListenersToJob(id, { resolve, reject }) + this.addListenersToJob(id, type,{ resolve, reject }) return } diff --git a/server/config.js b/server/config.js index eb4ebfc4..f1404a3f 100644 --- a/server/config.js +++ b/server/config.js @@ -27,7 +27,8 @@ module.exports = { /react-scripts/, /polymer-cli/, /^parcel$/, - /^devextreme$/ + /^devextreme$/, + /^yarn$/ ], unsupported: [ diff --git a/server/middlewares/rateLimit.middleware.js b/server/middlewares/rateLimit.middleware.js index 29f2f38e..06e2dbc3 100644 --- a/server/middlewares/rateLimit.middleware.js +++ b/server/middlewares/rateLimit.middleware.js @@ -8,7 +8,7 @@ const defaults = { blackList: [], accessLimited: '429: Too Many Requests.', accessForbidden: '403: This is forbidden area for you.', - max: 85, + max: 100, env: null, } diff --git a/server/middlewares/results/error.middleware.js b/server/middlewares/results/error.middleware.js index 56f93cd4..0ce528b3 100644 --- a/server/middlewares/results/error.middleware.js +++ b/server/middlewares/results/error.middleware.js @@ -41,7 +41,9 @@ async function errorHandler(ctx, next) { case 'BlacklistedPackageError': respondWithError(403, { code: 'BlacklistedPackageError', - message: 'The package you were looking for is blacklisted due to suspicious activity in the past', + message: 'The package you were looking for is blacklisted due to suspicious activity in the past, ' + + 'or because it\'s failed to build multiple times, ' + + 'and isn\'t bundle size calculations might not apply here.', }) break