Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gulp build improvements, update. #19405

Open
wants to merge 7 commits into
base: dev
Choose a base branch
from
Open

Gulp build improvements, update. #19405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ae4c06d
Bump to gulp 5
dannon Jan 13, 2025
d7a6562
Dependency lock
dannon Jan 13, 2025
8ee0d6a
Support --limit in gulpfile, to isolate particular plugins or sets fo…
dannon Jan 13, 2025
6d62ad1
Add specific target for fonts for easier tinkering
dannon Jan 21, 2025
384dc83
Remove fonts function from gulpfile and update client task
dannon Jan 21, 2025
714e21c
Drop out of date and unused fontawesome webfonts -- we don't use any …
dannon Jan 21, 2025
3a7b5fe
Adds state-saving for installed viz (plugin_build_hash) like we do for
dannon Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
234 changes: 134 additions & 100 deletions client/gulpfile.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ const xml2js = require("xml2js");
* un-built visualizations in the repository; for performance and
* simplicity just add them one at a time until we upgrade older viz's.
*/
const STATIC_PLUGIN_BUILD_IDS = [
const PLUGIN_BUILD_IDS = [
"annotate_image",
"chiraviz",
"drawrna",
Expand Down Expand Up @@ -45,8 +45,15 @@ const INSTALL_PLUGIN_BUILD_IDS = [
"venn",
"vizarr",
]; // todo: derive from XML
const DIST_PLUGIN_BUILD_IDS = ["new_user"];
const PLUGIN_BUILD_IDS = Array.prototype.concat(DIST_PLUGIN_BUILD_IDS, STATIC_PLUGIN_BUILD_IDS);

// Modify these lines to support the --limit option
const args = process.argv.slice(2);
const limitIndex = args.indexOf("--limit");
const pluginFilter = limitIndex !== -1 && args[limitIndex + 1] ? args[limitIndex + 1] : "";

const applyPluginFilter = (plugin) => {
return !pluginFilter || plugin.includes(pluginFilter);
};

const failOnError =
process.env.GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR && process.env.GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR !== "0"
Expand All @@ -56,9 +63,6 @@ const failOnError =
const PATHS = {
nodeModules: "./node_modules",
stagedLibraries: {
// This is a stepping stone towards having all this staged
// automatically. Eventually, this dictionary and staging step will
// not be necessary.
backbone: ["backbone.js", "backbone.js"],
jquery: ["dist/jquery.js", "jquery/jquery.js"],
"jquery-migrate": ["dist/jquery-migrate.js", "jquery/jquery.migrate.js"],
Expand All @@ -79,13 +83,15 @@ PATHS.pluginDirs = [
];

PATHS.pluginBuildModules = [
path.join(PATHS.pluginBaseDir, `{visualizations,welcome_page}/{${PLUGIN_BUILD_IDS.join(",")}}/package.json`),
...PLUGIN_BUILD_IDS.filter(applyPluginFilter).map((plugin) =>
path.join(PATHS.pluginBaseDir, `visualizations/${plugin}/package.json`)
),
];

function stageLibs(callback) {
Object.keys(PATHS.stagedLibraries).forEach((lib) => {
var p1 = path.resolve(path.join(PATHS.nodeModules, lib, PATHS.stagedLibraries[lib][0]));
var p2 = path.resolve(path.join("src", "libs", PATHS.stagedLibraries[lib][1]));
const p1 = path.resolve(path.join(PATHS.nodeModules, lib, PATHS.stagedLibraries[lib][0]));
const p2 = path.resolve(path.join("src", "libs", PATHS.stagedLibraries[lib][1]));
if (fs.existsSync(p1)) {
del.sync(p2);
fs.createReadStream(p1).pipe(fs.createWriteStream(p2));
Expand All @@ -99,12 +105,6 @@ function stageLibs(callback) {
return callback();
}

function fonts() {
return src(path.resolve(path.join(PATHS.nodeModules, "font-awesome/fonts/**/*"))).pipe(
dest("../static/images/fonts")
);
}

async function icons() {
await buildIcons("./src/assets/icons.json");
}
Expand All @@ -116,60 +116,48 @@ function stagePlugins() {
function buildPlugins(callback, forceRebuild) {
/*
* Walk pluginBuildModules glob and attempt to build modules.
* */
*/
const packageJsons = globSync(PATHS.pluginBuildModules, {});
packageJsons.forEach((file) => {
let skipBuild = false;
const pluginDir = path.dirname(file);
const pluginName = pluginDir.split(path.sep).pop();

const hashFilePath = path.join(
pluginDir,
DIST_PLUGIN_BUILD_IDS.indexOf(pluginName) > -1 ? "dist" : "static",
"plugin_build_hash.txt"
);
const hashFilePath = path.join(pluginDir, "static", "plugin_build_hash.txt");

if (forceRebuild) {
skipBuild = false;
} else {
// Try reading existing plugin_build_hash.txt
if (fs.existsSync(hashFilePath)) {
const hashFileContent = fs.readFileSync(hashFilePath, "utf8").trim();
const isHash = /^[0-9a-f]{7,40}$/.test(hashFileContent); // Check for a 7 to 40 character hexadecimal string
const oldHash = fs.readFileSync(hashFilePath, "utf8").trim();
const isHash = /^[0-9a-f]{7,40}$/.test(oldHash);

if (!isHash) {
console.log(`Hash file for ${pluginName} exists but does not have a valid git hash.`);
skipBuild = false;
} else {
skipBuild =
child_process.spawnSync("git", ["diff", "--quiet", hashFileContent, "--", pluginDir], {
stdio: "inherit",
shell: true,
}).status === 0;
}

skipBuild =
child_process.spawnSync("git", ["diff", "--quiet", `$(cat ${hashFilePath})`, "--", pluginDir], {
// Check if there are changes since the stored hash
const diffResult = child_process.spawnSync("git", ["diff", "--quiet", oldHash, "--", pluginDir], {
stdio: "inherit",
shell: true,
}).status === 0;
if (!skipBuild) {
// Hash exists and is outdated, triggering a rebuild.
// Stage current hash to .orig for debugging and to
// force a plugin rebuild in the event of a failure
// (i.e. -- we're committed to a new build of this plugin).
fs.renameSync(hashFilePath, `${hashFilePath}.orig`);
});
skipBuild = diffResult.status === 0;
if (!skipBuild) {
// Hash exists and is outdated, rename to .orig
fs.renameSync(hashFilePath, `${hashFilePath}.orig`);
}
}
} else {
console.log(`No build hashfile detected for ${pluginName}, generating now.`);
}
}

if (skipBuild) {
console.log(`No changes detected for ${pluginName}`);
console.log(`No changes detected for ${pluginName}, skipping build.`);
} else {
console.log(`Installing Dependencies for ${pluginName}`);

// Else we call yarn install and yarn build
// Yarn install
child_process.spawnSync(
"yarn",
["install", "--production=false", "--network-timeout=300000", "--check-files"],
Expand All @@ -179,32 +167,34 @@ function buildPlugins(callback, forceRebuild) {
shell: true,
}
);

console.log(`Building ${pluginName}`);
const opts = {
cwd: pluginDir,
stdio: "inherit",
shell: true,
};
// if node version is >16, set NODE_OPTIONS to use legacy openssl provider
if (process.versions.node.split(".")[0] > "16") {
// Node >16 fix
if (Number(process.versions.node.split(".")[0]) > 16) {
opts.env = {
...process.env,
PARCEL_WORKER_BACKEND: "process",
NODE_OPTIONS: "--openssl-legacy-provider",
};
}
if (child_process.spawnSync("yarn", ["build"], opts).status === 0) {

const buildResult = child_process.spawnSync("yarn", ["build"], opts);
if (buildResult.status === 0) {
console.log(`Successfully built, saving build state to ${hashFilePath}`);
child_process.exec(`(git rev-parse HEAD 2>/dev/null || echo \`\`) > ${hashFilePath}`);
// Save new hash
child_process.execSync(`git rev-parse HEAD > ${hashFilePath}`, {
stdio: "inherit",
shell: true,
});
} else {
console.error(
`Error building ${pluginName}, not saving build state. Please report this issue to the Galaxy Team.`
);
console.error(`Error building ${pluginName}, not saving build state.`);
if (failOnError) {
// Fail on error.
console.error(
"Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set, see error(s) above."
);
console.error("Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set.");
process.exit(1);
}
}
Expand All @@ -213,75 +203,119 @@ function buildPlugins(callback, forceRebuild) {
return callback();
}

async function installPlugins(callback) {
// iterate through install_plugin_build_ids, identify xml files and install dependencies
for (const plugin_name of INSTALL_PLUGIN_BUILD_IDS) {
const pluginDir = path.join(PATHS.pluginBaseDir, `visualizations/${plugin_name}`);
const xmlPath = path.join(pluginDir, `config/${plugin_name}.xml`);
// Check if the file exists
if (fs.existsSync(xmlPath)) {
await installDependenciesFromXML(xmlPath, pluginDir);
} else {
/**
* Same logic as buildPlugins, but for 'installed' plugins that rely on an XML
* specifying npm dependencies (the INSTALL_PLUGIN_BUILD_IDS).
*/
async function installPlugins(callback, forceReinstall) {
for (const pluginName of INSTALL_PLUGIN_BUILD_IDS.filter(applyPluginFilter)) {
const pluginDir = path.join(PATHS.pluginBaseDir, `visualizations/${pluginName}`);
const xmlPath = path.join(pluginDir, `config/${pluginName}.xml`);

if (!fs.existsSync(xmlPath)) {
console.error(`XML file not found: ${xmlPath}`);
continue;
}

const hashFilePath = path.join(pluginDir, "static", "plugin_build_hash.txt");

let skipInstall = false;
if (!forceReinstall && fs.existsSync(hashFilePath)) {
const oldHash = fs.readFileSync(hashFilePath, "utf8").trim();
const isHash = /^[0-9a-f]{7,40}$/.test(oldHash);

if (!isHash) {
console.log(`Hash file for ${pluginName} exists but is not a valid git hash.`);
} else {
const diffResult = child_process.spawnSync("git", ["diff", "--quiet", oldHash, "--", pluginDir], {
stdio: "inherit",
shell: true,
});
skipInstall = diffResult.status === 0;
if (!skipInstall) {
fs.renameSync(hashFilePath, `${hashFilePath}.orig`);
}
}
} else if (!fs.existsSync(hashFilePath)) {
console.log(`No install hashfile detected for ${pluginName}, generating now.`);
}

if (skipInstall) {
console.log(`No changes detected for ${pluginName}, skipping re-installation.`);
} else {
console.log(`Installing npm dependencies for installed plugin ${pluginName}...`);
try {
await installDependenciesFromXML(xmlPath, pluginDir);
console.log(`Finished installing for ${pluginName}, saving new hash.`);

// Save the current commit hash
child_process.execSync(`git rev-parse HEAD > ${hashFilePath}`, {
stdio: "inherit",
shell: true,
});
} catch (err) {
console.error(`Error installing for ${pluginName}:`, err);
if (failOnError) {
console.error(
"Failing build due to GALAXY_PLUGIN_BUILD_FAIL_ON_ERROR being set, see error(s) above."
);
process.exit(1);
}
}
}
}
return callback();
}

// Function to parse the XML and install dependencies
// Function to parse XML and install specified npm packages
async function installDependenciesFromXML(xmlPath, pluginDir) {
try {
const pluginXML = fs.readFileSync(xmlPath);
const parsedXML = await xml2js.parseStringPromise(pluginXML);
const requirements = parsedXML.visualization.requirements[0].requirement;

const installPromises = requirements.map(async (dep) => {
const { type: reqType, package: pkgName, version } = dep.$;

if (reqType === "npm" && pkgName && version) {
try {
const installResult = child_process.spawnSync(
"npm",
["install", "--silent", "--no-save", "--prefix .", `${pkgName}@${version}`],
{
cwd: pluginDir,
stdio: "inherit",
shell: true,
}
);
const pluginXML = fs.readFileSync(xmlPath);
const parsedXML = await xml2js.parseStringPromise(pluginXML);
const requirements = parsedXML?.visualization?.requirements?.[0]?.requirement || [];

if (installResult.status === 0) {
await fs.copy(
path.join(pluginDir, "node_modules", pkgName, "static"),
path.join(pluginDir, "static")
);
console.log(`Installed package ${pkgName}@${version} in ${pluginDir}`);
} else {
console.error(`Error installing package ${pkgName}@${version} in ${pluginDir}`);
}
} catch (err) {
console.error(`Error handling package ${pkgName}@${version} in ${pluginDir}:`, err);
// For each <requirement type="npm" package="xxx" version="yyy" />
for (const dep of requirements) {
const { type: reqType, package: pkgName, version } = dep.$;
if (reqType === "npm" && pkgName && version) {
console.log(` → Installing ${pkgName}@${version} in ${pluginDir}`);
const installResult = child_process.spawnSync(
"npm",
["install", "--silent", "--no-save", `--prefix=${pluginDir}`, `${pkgName}@${version}`],
{
cwd: pluginDir,
stdio: "inherit",
shell: true,
}
}
});
);

await Promise.all(installPromises);
} catch (err) {
console.error(`Error processing XML file ${xmlPath}:`, err);
if (installResult.status === 0) {
// Installed successfully, copy over '/static' assets
const potentialStaticPath = path.join(pluginDir, "node_modules", pkgName, "static");
if (fs.existsSync(potentialStaticPath)) {
await fs.copy(potentialStaticPath, path.join(pluginDir, "static"));
}
} else {
throw new Error(`npm install of ${pkgName}@${version} failed in ${pluginDir}`);
}
}
}
}

function forceBuildPlugins(callback) {
return buildPlugins(callback, true);
}

function forceInstallPlugins(callback) {
return installPlugins(callback, true);
}

function cleanPlugins() {
return del(["../static/plugins/{visualizations,welcome_page}/*"], { force: true });
}

const client = parallel(fonts, stageLibs, icons);
const client = parallel(stageLibs, icons);
const plugins = series(buildPlugins, installPlugins, cleanPlugins, stagePlugins);
const pluginsRebuild = series(forceBuildPlugins, installPlugins, cleanPlugins, stagePlugins);
const pluginsRebuild = series(forceBuildPlugins, forceInstallPlugins, cleanPlugins, stagePlugins);

function watchPlugins() {
const BUILD_PLUGIN_WATCH_GLOB = [
Expand Down
2 changes: 1 addition & 1 deletion client/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@
"expose-loader": "^4.1.0",
"fake-indexeddb": "^6.0.0",
"fs-extra": "^11.2.0",
"gulp": "^4.0.2",
"gulp": "^5.0.0",
"ignore-loader": "^0.1.2",
"imports-loader": "^4.0.1",
"jest": "^29.7.0",
Expand Down
Loading
Loading