generate-index: Use west to fetch nrf revision

VSC-2486 Add ncs revision to schemas. Add filtering by ncs version to query params. Add filtering by ncs version to webpage. Fetch nrf-sdk version using west. Signed-off-by: Filip Zajdel <[email protected]>
nrfconnect · Mar 25, 2024 · 9842085 · 9842085
1 parent 391e851
commit 9842085
Show file tree

Hide file tree

Showing 8 changed files with 193 additions and 38 deletions.
diff --git a/resources/output_schema.json b/resources/output_schema.json
@@ -176,6 +176,9 @@
                     "defaultBranch": {
                         "type": "string"
                     },
+                    "defaultNcs" : {
+                        "type": "string"
+                    },
                     "lastUpdate": {
                         "type": "string",
                         "format": "date-time"

diff --git a/scripts/generate-index-json.ts b/scripts/generate-index-json.ts
@@ -11,6 +11,8 @@ import path from 'path';
 import { Octokit } from '@octokit/rest';
 import fetch from 'node-fetch';
 import colours from 'ansi-colors';
+import * as yaml from "yaml";
+import WestEnv from './west';
 
 import type {
     OrgIndex,
@@ -24,6 +26,7 @@ import { execSync } from 'child_process';
 
 const nordicOrgs: string[] = ['nrfconnect', 'nordic', 'nordicplayground'];
 const partnerOrgs: string[] = ['golioth'];
+const tempAppDir: string = 'apps';
 
 function notUndefined<T>(value: T | undefined): value is T {
     return value !== undefined;
@@ -122,6 +125,32 @@ async function fetchRepoData(
             repo: app.name,
         });
 
+        const repoUrl = `https://github.com/${orgId}/${app.name}`;
+
+        const westEnv = new WestEnv(repoUrl, repoData.default_branch, `${tempAppDir}/${app.name}`);
+
+        const ncsVersions: Map<string, string> = new Map();
+
+        try {
+            await westEnv.init();
+            const nrfRevision = await westEnv.getModuleRev("nrf");
+            ncsVersions.set(repoData.default_branch, nrfRevision ?? "");
+            console.log(`nrfRevision: ${nrfRevision}`);
+        } catch (e) {
+            console.log(`West env failed: ${e}`);
+        }
+
+        const revs = [...releases.data.map((release) => release.tag_name)];
+        for (const rev of revs) {
+            try {
+                await westEnv.checkout(rev);
+                const ncsVer = await westEnv.getModuleRev("nrf") ?? "";
+                ncsVersions.set(rev, ncsVer);
+            } catch (e) {
+                console.log(`Checkout failed: ${e}`);
+            }
+        }
+
         console.log(colours.green(`Fetched data for ${orgId}/${app.name}`));
 
         return {
@@ -132,6 +161,7 @@ async function fetchRepoData(
             name: app.name,
             title: app.title,
             defaultBranch: repoData.default_branch,
+            defaultNcs: ncsVersions.get(repoData.default_branch) ?? "",
             isTemplate: repoData.is_template ?? false,
             kind: app.kind,
             lastUpdate: repoData.updated_at,
@@ -144,6 +174,7 @@ async function fetchRepoData(
                 date: release.created_at,
                 name: release.name ?? release.tag_name,
                 tag: release.tag_name,
+                ncs: ncsVersions.get(release.tag_name) ?? ""
             })),
             tags: app.tags,
         };
@@ -152,12 +183,22 @@ async function fetchRepoData(
     }
 }
 
+async function cleanup() {
+    try {
+        await fs.access(tempAppDir);
+        await fs.rm(tempAppDir, { recursive: true});
+    } catch {
+        // No need to remove the directory
+    }
+}
+
 async function run() {
     const orgIndices = await readOrgIndexFiles();
     const appIndex = await generateIndex(orgIndices);
     const stringified = JSON.stringify(appIndex, undefined, 2);
     const indexPath = path.join(__dirname, '..', 'resources', 'index.json');
     await fs.writeFile(indexPath, stringified);
+    await cleanup();
     console.log(`\nWritten app index to ${indexPath}`);
 }
 

diff --git a/scripts/west.ts b/scripts/west.ts
@@ -0,0 +1,63 @@
+/* Copyright (c) 2023 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+import { exec } from 'child_process';
+import fs from 'fs/promises';
+import { promisify } from 'util';
+
+const execAsync = promisify(exec);
+
+async function exists(path: string): Promise<boolean> {
+    try {
+        await fs.access(path);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+class WestEnv {
+
+    constructor(private repoUrl: string, private revision: string, private appPath: string) {}
+
+    async init(): Promise<void> {
+        try {
+            const dirExists = await exists(this.appPath);
+            if (!dirExists) {
+                await fs.mkdir(this.appPath, { recursive: true });
+            }
+            await execAsync(
+                `cd ${this.appPath} && west init -m ${this.repoUrl} --mr ${this.revision} && west update`
+            )
+        } catch(err) {
+            Promise.reject(`west init failed with err: ${err}`);
+        }
+    }
+
+    async checkout(revision: string): Promise<void> {
+
+        const {stdout} = await execAsync(`cd ${this.appPath} && west config manifest.path`);
+        const manifestRepoPath = `${this.appPath}/${stdout.trim()}`;
+
+        await execAsync(`cd ${manifestRepoPath} && git checkout ${revision} && west update`)
+
+        this.revision = revision;
+    }
+
+    get manifestRev(): string {
+        return this.revision;
+    }
+
+    async getModuleRev(module: string): Promise<string | undefined> {
+
+        let {stdout} = await execAsync(
+            `cd ${this.appPath} && west list --all --format "{name};{revision}" | grep "${module};"`
+        );
+
+        return stdout.split(";")[1]?.trim();
+    }
+}
+
+export default WestEnv;
diff --git a/site/src/app/Header.tsx b/site/src/app/Header.tsx
@@ -39,8 +39,8 @@ interface Props {
 }
 
 function Header(props: Props): JSX.Element {
-    function handleTextSearchChange(e: ChangeEvent<HTMLInputElement>) {
-        props.dispatchFilters({ type: 'textSearch', payload: e.target.value });
+    function handleSearch(type: 'appSearch' | 'ncsSearch') {
+        return (e: ChangeEvent<HTMLInputElement>) => { props.dispatchFilters({ type: type, payload: e.target.value }); }
     }
 
     const aboutIcon = (
@@ -94,25 +94,44 @@ function Header(props: Props): JSX.Element {
                     </div>
 
                     <div className="absolute bottom-0 flex w-full justify-center">
-                        <input
-                            type="search"
-                            placeholder="Filter applications..."
-                            value={props.filters.textSearch}
-                            onChange={handleTextSearchChange}
-                            aria-label="Filter applications"
-                            className="relative top-5 mx-4 h-14 w-full max-w-5xl p-3 pl-3 outline-none drop-shadow-md lg:mx-0 lg:w-2/3"
-                        />
+                        <div className="relative flex w-2/3 relative top-5 mx-4 h-14 max-w-5xl">
+                            <input
+                                type="search"
+                                placeholder="Filter applications..."
+                                value={props.filters.appSearch}
+                                onChange={handleSearch("appSearch")}
+                                aria-label="Filter applications"
+                                className="p-3 pl-3 outline-none drop-shadow-md lg:mx-0 lg:w-3/4"
+                            />
+                            <input
+                                type="search"
+                                placeholder="NCS version..."
+                                value={props.filters.ncsSearch}
+                                onChange={handleSearch("ncsSearch")}
+                                aria-label="Filter applications"
+                                className="p-3 pl-3 outline-none drop-shadow-md lg:mx-0 lg:w-1/4"
+                            />
+                        </div>
                     </div>
                 </div>
             </header>
 
-            <input
-                type="text"
-                placeholder="Filter applications..."
-                value={props.filters.textSearch}
-                onChange={handleTextSearchChange}
-                className="h-14 w-full rounded-none border-b border-gray-300 pl-3 md:hidden"
-            />
+            <div className="h-14 w-full rounded-none md:hidden">
+                <input
+                    type="text"
+                    placeholder="Filter applications..."
+                    value={props.filters.appSearch}
+                    onChange={handleSearch("appSearch")}
+                    className="h-full w-2/3 border-b border-r border-gray-300 pl-3"
+                />
+                <input
+                    type="text"
+                    placeholder="NCS ..."
+                    value={props.filters.ncsSearch}
+                    onChange={handleSearch("ncsSearch")}
+                    className="h-full w-1/3 border-b border-gray-300 pl-3"
+                />
+            </div>
         </div>
     );
 }

diff --git a/site/src/app/Root.tsx b/site/src/app/Root.tsx
@@ -51,10 +51,15 @@ function Root(props: Props) {
 
     useEffect(() => {
         const searchParams = new URLSearchParams(window.location.search);
-        const appFilter = searchParams.get("app");
+        const app = searchParams.get("app");
+        const ncs = searchParams.get("ncs");
 
-        if (appFilter) {
-            dispatchFilters({ type: 'textSearch', payload: appFilter });
+        if (app) {
+            dispatchFilters({ type: 'appSearch', payload: app });
+        }
+
+        if (ncs) {
+            dispatchFilters({ type: 'ncsSearch', payload: ncs });
         }
     }, []);
 

diff --git a/site/src/app/filters.ts b/site/src/app/filters.ts
@@ -7,39 +7,60 @@ import Fuse from 'fuse.js';
 
 import { NormalisedApp } from '../schema';
 
-export interface Filters {
-    textSearch: string;
+
+function filterAppName(apps: NormalisedApp[], search: string): NormalisedApp[] {
+    const fuse = new Fuse(apps, { keys: ['owner.name', 'title', 'description', 'name', 'tags'] });
+    const results = fuse.search(search);
+    return results.map((result) => result.item);
+}
+
+function filterNcsVersion(apps: NormalisedApp[], search: string): NormalisedApp[] {
+    const exactMatches: NormalisedApp[] = [];
+
+    for (const app of apps) {
+        for (const release of app.releases) {
+            if (release.ncs.includes(search)) {
+                exactMatches.push(app);
+                break;
+            }
+        }
+    }
+
+    return exactMatches;
 }
 
-export const initialFilters: Filters = {
-    textSearch: '',
+export interface Filters {
+    appSearch: string;
+    ncsSearch: string;
 };
 
+export const initialFilters: Filters = { appSearch: "", ncsSearch: "" };
+
 interface TextSearchAction {
-    type: 'textSearch';
+    type: 'ncsSearch' | 'appSearch';
     payload: string;
 }
 
 export type FilterAction = TextSearchAction;
 
-export function filterReducer(state: Filters, action: FilterAction): Filters {
+export function filterReducer(filters: Filters, action: FilterAction): Filters {
     switch (action.type) {
-        case 'textSearch':
-            return { ...state, textSearch: action.payload };
+        case 'appSearch':
+            return { ...filters, appSearch: action.payload};
+        case 'ncsSearch':
+            return { ...filters, ncsSearch: action.payload};
     }
 }
 
 export function filterApps(apps: NormalisedApp[], filters: Filters): NormalisedApp[] {
-    if (!filters.textSearch) {
-        return apps;
+
+    if (filters.appSearch !== '') {
+        apps = filterAppName(apps, filters.appSearch);
     }
 
-    filters = normaliseFilters(filters);
-    const fuse = new Fuse(apps, { keys: ['owner.name', 'title', 'description', 'name', 'tags'] });
-    const results = fuse.search(filters.textSearch);
-    return results.map((result) => result.item);
-}
+    if (filters.ncsSearch !== '') {
+        apps = filterNcsVersion(apps, filters.ncsSearch);
+    }
 
-function normaliseFilters(filters: Filters): Filters {
-    return { ...filters, textSearch: filters.textSearch.toLowerCase() };
+    return apps;
 }
diff --git a/site/src/sampleData.ts b/site/src/sampleData.ts
@@ -51,6 +51,7 @@ function createFakeApp(): AppIndex['apps'][number] {
                 name: faker.system.semver(),
                 date: faker.date.recent().toString(),
                 tag: faker.git.branch(),
+                ncs: faker.git.branch(),
             }),
             { count: { min: 1, max: 5 } },
         ),

diff --git a/site/src/schema.ts b/site/src/schema.ts
@@ -170,8 +170,9 @@ export const appSchema = {
                     tag: { type: 'string' },
                     name: { type: 'string' },
                     date: { type: 'string', format: 'date' },
+                    ncs: { type: 'string'},
                 },
-                required: ['tag', 'name', 'date'],
+                required: ['tag', 'name', 'date', 'ncs'],
                 additionalProperties: false,
             },
             minItems: 1,
@@ -180,6 +181,7 @@ export const appSchema = {
         stars: { type: 'integer' },
         forks: { type: 'integer' },
         defaultBranch: { type: 'string' },
+        defaultNcs: { type: 'string' },
         lastUpdate: { type: 'string', format: 'date-time' },
         apps: { type: 'string' },
     },